From cc17c4a45849957f36cc0dd63ccb34bfec0fa785 Mon Sep 17 00:00:00 2001 From: Pablo Curiel Date: Mon, 4 Jul 2022 00:14:03 +0200 Subject: [PATCH] Implement layered NCA storage interface. Updating PFS and RomFS comes next. Also fixed Sparse storages not being supported as an indirect storage's original substorage. --- include/core/bktr.h | 6 +- include/core/nca_storage.h | 68 ++++++++++ source/core/bktr.c | 16 +-- source/core/nca_storage.c | 252 +++++++++++++++++++++++++++++++++++++ 4 files changed, 332 insertions(+), 10 deletions(-) create mode 100644 include/core/nca_storage.h create mode 100644 source/core/nca_storage.c diff --git a/include/core/bktr.h b/include/core/bktr.h index a1fc157..ee9877b 100644 --- a/include/core/bktr.h +++ b/include/core/bktr.h @@ -154,10 +154,10 @@ typedef enum { ///< May be used as substorage for all BucketTreeStorage types. BucketTreeSubStorageType_Indirect = 1, ///< Indirect storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Compressed only. BucketTreeSubStorageType_AesCtrEx = 2, ///< AesCtrEx storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Indirect only. - BucketTreeSubStorageType_Sparse = 3, ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Used in base applications only. - ///< May be used as substorage for BucketTreeStorageType_Compressed only. - BucketTreeSubStorageType_Compressed = 4, ///< Compressed storage. If available, this is always the outmost storage type for any NCA. May be used by all title types. + BucketTreeSubStorageType_Compressed = 3, ///< Compressed storage. If available, this is always the outmost storage type for any NCA. May be used by all title types. ///< May be used as substorage for BucketTreeStorageType_Indirect only. + BucketTreeSubStorageType_Sparse = 4, ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Used in base applications only. + ///< May be used as substorage for BucketTreeStorageType_Compressed or BucketTreeStorageType_Indirect. BucketTreeSubStorageType_Count = 5 ///< Total values supported by this enum. } BucketTreeSubStorageType; diff --git a/include/core/nca_storage.h b/include/core/nca_storage.h new file mode 100644 index 0000000..d6ef9af --- /dev/null +++ b/include/core/nca_storage.h @@ -0,0 +1,68 @@ +/* + * nca_storage.h + * + * Copyright (c) 2020-2022, DarkMatterCore . + * + * This file is part of nxdumptool (https://github.com/DarkMatterCore/nxdumptool). + * + * nxdumptool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nxdumptool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#ifndef __NCA_STORAGE_H__ +#define __NCA_STORAGE_H__ + +#include "bktr.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + NcaStorageBaseStorageType_Invalid = 0, /* Placeholder. */ + NcaStorageBaseStorageType_Regular = 1, + NcaStorageBaseStorageType_Sparse = 2, + NcaStorageBaseStorageType_Indirect = 3, + NcaStorageBaseStorageType_Compressed = 4 +} NcaStorageBaseStorageType; + +/// Used to perform multi-layer reads within a single NCA FS section. +typedef struct { + u8 base_storage_type; ///< NcaStorageBaseStorageType. + NcaFsSectionContext *nca_fs_ctx; ///< NCA FS section context used to initialize this context. + BucketTreeContext *sparse_storage; ///< Sparse storage context. + BucketTreeContext *aes_ctr_ex_storage; ///< AesCtrEx storage context. + BucketTreeContext *indirect_storage; ///< Indirect storage context. + BucketTreeContext *compressed_storage; ///< Compressed storage context. +} NcaStorageContext; + +/// Initializes a NCA storage context using a NCA FS section context. +bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx); + +/// Sets a storage from the provided Base NcaStorageContext as the original substorage for the provided Patch NcaStorageContext's Indirect Storage. +/// Needed to perform combined reads between a base NCA and a patch NCA. +bool ncaStorageSetPatchOriginalSubStorage(NcaStorageContext *patch_ctx, NcaStorageContext *base_ctx); + +/// Reads data from the NCA storage using a previously initialized NcaStorageContext. +bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset); + +/// Frees a previously initialized NCA storage context. +void ncaStorageFreeContext(NcaStorageContext *ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* __NCA_STORAGE_H__ */ diff --git a/source/core/bktr.c b/source/core/bktr.c index e05e536..149c69c 100644 --- a/source/core/bktr.c +++ b/source/core/bktr.c @@ -198,13 +198,14 @@ bool bktrSetRegularSubStorage(BucketTreeContext *ctx, NcaFsSectionContext *nca_f bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContext *child_ctx, u8 substorage_index) { if (!bktrIsValidContext(parent_ctx) || !bktrIsValidContext(child_ctx) || substorage_index >= BKTR_MAX_SUBSTORAGE_COUNT || \ - (parent_ctx->storage_type != BucketTreeStorageType_Indirect && substorage_index != 0) || \ - (parent_ctx->storage_type == BucketTreeStorageType_Indirect && child_ctx->storage_type != BucketTreeStorageType_Compressed && child_ctx->storage_type != BucketTreeStorageType_AesCtrEx) || \ - (parent_ctx->storage_type == BucketTreeStorageType_Indirect && child_ctx->storage_type == BucketTreeStorageType_Compressed && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx)) || \ - (parent_ctx->storage_type == BucketTreeStorageType_Indirect && child_ctx->storage_type == BucketTreeStorageType_AesCtrEx && (substorage_index != 1 || parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) || \ parent_ctx->storage_type == BucketTreeStorageType_AesCtrEx || parent_ctx->storage_type == BucketTreeStorageType_Sparse || \ - (parent_ctx->storage_type == BucketTreeStorageType_Compressed && child_ctx->storage_type != BucketTreeStorageType_Indirect && child_ctx->storage_type != BucketTreeStorageType_Sparse) || \ - (parent_ctx->storage_type == BucketTreeStorageType_Compressed && parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) + (parent_ctx->storage_type != BucketTreeStorageType_Indirect && substorage_index != 0) || \ + (parent_ctx->storage_type == BucketTreeStorageType_Indirect && (child_ctx->storage_type < BucketTreeStorageType_AesCtrEx || \ + child_ctx->storage_type > BucketTreeStorageType_Sparse || (child_ctx->storage_type == BucketTreeStorageType_AesCtrEx && (substorage_index != 1 || \ + parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) || ((child_ctx->storage_type == BucketTreeStorageType_Compressed || \ + child_ctx->storage_type == BucketTreeStorageType_Sparse) && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx)))) || \ + (parent_ctx->storage_type == BucketTreeStorageType_Compressed && ((child_ctx->storage_type != BucketTreeStorageType_Indirect && \ + child_ctx->storage_type != BucketTreeStorageType_Sparse) || parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx))) { LOG_MSG("Invalid parameters!"); return false; @@ -364,7 +365,8 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r bool is_sparse = (ctx->storage_type == BucketTreeStorageType_Sparse); if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || (!is_sparse && !bktrIsValidSubstorage(&(ctx->substorages[1]))) || \ - (!is_sparse && ((ctx->substorages[0].type != BucketTreeSubStorageType_Regular && ctx->substorages[0].type != BucketTreeStorageType_Compressed) || ctx->substorages[1].type != BucketTreeSubStorageType_AesCtrEx)) || \ + (!is_sparse && ((ctx->substorages[0].type != BucketTreeSubStorageType_Regular && ctx->substorages[0].type != BucketTreeStorageType_Compressed && \ + ctx->substorages[0].type != BucketTreeSubStorageType_Sparse) || ctx->substorages[1].type != BucketTreeSubStorageType_AesCtrEx)) || \ (is_sparse && ctx->substorages[0].type != BucketTreeSubStorageType_Regular)) { LOG_MSG("Invalid parameters!"); diff --git a/source/core/nca_storage.c b/source/core/nca_storage.c new file mode 100644 index 0000000..23e53c5 --- /dev/null +++ b/source/core/nca_storage.c @@ -0,0 +1,252 @@ +/* + * nca_storage.c + * + * Copyright (c) 2020-2022, DarkMatterCore . + * + * This file is part of nxdumptool (https://github.com/DarkMatterCore/nxdumptool). + * + * nxdumptool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nxdumptool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nxdt_utils.h" +#include "nca_storage.h" + +/* Function prototypes. */ + +static bool ncaStorageInitializeBucketTreeContext(BucketTreeContext **out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type); +NX_INLINE bool ncaStorageIsValidContext(NcaStorageContext *ctx); + +bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx) +{ + /* TODO: allow patches with sparse layers? */ + if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || (nca_fs_ctx->has_sparse_layer && nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs)) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + bool success = false; + + /* Free output context beforehand. */ + ncaStorageFreeContext(out); + + /* Set initial base storage type. */ + out->base_storage_type = NcaStorageBaseStorageType_Regular; + + /* Check if a sparse layer is available. */ + if (nca_fs_ctx->has_sparse_layer) + { + /* Initialize sparse layer. */ + if (!ncaStorageInitializeBucketTreeContext(&(out->sparse_storage), nca_fs_ctx, BucketTreeStorageType_Sparse)) goto end; + + /* Set sparse layer's substorage. */ + if (!bktrSetRegularSubStorage(out->sparse_storage, nca_fs_ctx)) goto end; + + /* Update base storage type. */ + out->base_storage_type = NcaStorageBaseStorageType_Sparse; + } + + /* Check if both Indirect and AesCtrEx layers are available. */ + if (nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs) + { + /* Initialize AesCtrEx and Indirect layers. */ + if (!ncaStorageInitializeBucketTreeContext(&(out->aes_ctr_ex_storage), nca_fs_ctx, BucketTreeStorageType_AesCtrEx) || \ + !ncaStorageInitializeBucketTreeContext(&(out->indirect_storage), nca_fs_ctx, BucketTreeStorageType_Indirect)) goto end; + + /* Set AesCtrEx layer's substorage. */ + if (!bktrSetRegularSubStorage(out->aes_ctr_ex_storage, nca_fs_ctx)) goto end; + + /* Set Indirect layer's AesCtrEx substorage. */ + /* Base substorage must be manually set at a later time. */ + if (!bktrSetBucketTreeSubStorage(out->indirect_storage, out->aes_ctr_ex_storage, 1)) goto end; + + /* Update base storage type. */ + out->base_storage_type = NcaStorageBaseStorageType_Indirect; + } + + /* Check if a compression layer is available. */ + if (nca_fs_ctx->has_compression_layer) + { + /* Initialize compression layer. */ + if (!ncaStorageInitializeBucketTreeContext(&(out->compressed_storage), nca_fs_ctx, BucketTreeStorageType_Compressed)) goto end; + + /* Set compression layer's substorage. */ + switch(out->base_storage_type) + { + case NcaStorageBaseStorageType_Regular: + if (!bktrSetRegularSubStorage(out->compressed_storage, nca_fs_ctx)) goto end; + break; + case NcaStorageBaseStorageType_Sparse: + if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->sparse_storage, 0)) goto end; + break; + case NcaStorageBaseStorageType_Indirect: + if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->indirect_storage, 0)) goto end; + break; + } + + /* Update base storage type. */ + out->base_storage_type = NcaStorageBaseStorageType_Compressed; + } + + /* Update return value. */ + success = true; + +end: + if (!success) ncaStorageFreeContext(out); + + return success; +} + +bool ncaStorageSetPatchOriginalSubStorage(NcaStorageContext *patch_ctx, NcaStorageContext *base_ctx) +{ + NcaContext *patch_nca_ctx = NULL, *base_nca_ctx = NULL; + + if (!ncaStorageIsValidContext(patch_ctx) || !ncaStorageIsValidContext(base_ctx) || patch_ctx->nca_fs_ctx == base_ctx->nca_fs_ctx || \ + !(patch_nca_ctx = (NcaContext*)patch_ctx->nca_fs_ctx->nca_ctx) || !(base_nca_ctx = (NcaContext*)base_ctx->nca_fs_ctx->nca_ctx) || \ + patch_ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || base_ctx->nca_fs_ctx->section_type != NcaFsSectionType_RomFs || \ + patch_nca_ctx->header.program_id != base_nca_ctx->header.program_id || patch_nca_ctx->header.content_type != base_nca_ctx->header.content_type || \ + patch_nca_ctx->id_offset != base_nca_ctx->id_offset || patch_nca_ctx->title_version <= base_nca_ctx->title_version || !patch_ctx->indirect_storage) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + bool success = false; + + /* Set base storage. */ + switch(base_ctx->base_storage_type) + { + case NcaStorageBaseStorageType_Regular: + success = bktrSetRegularSubStorage(patch_ctx->indirect_storage, base_ctx->nca_fs_ctx); + break; + case NcaStorageBaseStorageType_Sparse: + success = bktrSetBucketTreeSubStorage(patch_ctx->indirect_storage, base_ctx->sparse_storage, 0); + break; + case NcaStorageBaseStorageType_Compressed: + success = bktrSetBucketTreeSubStorage(patch_ctx->indirect_storage, base_ctx->compressed_storage, 0); + break; + default: + break; + } + + if (!success) LOG_MSG("Failed to set base storage to patch storage!"); + + return success; +} + +bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset) +{ + if (!ncaStorageIsValidContext(ctx) || !out || !read_size) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + bool success = false; + + switch(ctx->base_storage_type) + { + case NcaStorageBaseStorageType_Regular: + success = ncaReadFsSection(ctx->nca_fs_ctx, out, read_size, offset); + break; + case NcaStorageBaseStorageType_Sparse: + success = bktrReadStorage(ctx->sparse_storage, out, read_size, offset); + break; + case NcaStorageBaseStorageType_Indirect: + success = bktrReadStorage(ctx->indirect_storage, out, read_size, offset); + break; + case NcaStorageBaseStorageType_Compressed: + success = bktrReadStorage(ctx->compressed_storage, out, read_size, offset); + break; + default: + break; + } + + if (!success) LOG_MSG("Failed to read 0x%lX-byte long block from offset 0x%lX in base storage! (%u).", read_size, offset, ctx->base_storage_type); + + return success; +} + +void ncaStorageFreeContext(NcaStorageContext *ctx) +{ + if (!ctx) return; + + if (ctx->sparse_storage) + { + bktrFreeContext(ctx->sparse_storage); + free(ctx->sparse_storage); + } + + if (ctx->aes_ctr_ex_storage) + { + bktrFreeContext(ctx->aes_ctr_ex_storage); + free(ctx->aes_ctr_ex_storage); + } + + if (ctx->indirect_storage) + { + bktrFreeContext(ctx->indirect_storage); + free(ctx->indirect_storage); + } + + if (ctx->compressed_storage) + { + bktrFreeContext(ctx->compressed_storage); + free(ctx->compressed_storage); + } + + memset(ctx, 0, sizeof(NcaStorageContext)); +} + +static bool ncaStorageInitializeBucketTreeContext(BucketTreeContext **out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type) +{ + if (!out || !nca_fs_ctx || storage_type >= BucketTreeStorageType_Count) + { + LOG_MSG("Invalid parameters!"); + return false; + } + + BucketTreeContext *bktr_ctx = NULL; + bool success = false; + + /* Allocate memory for the Bucket Tree context. */ + bktr_ctx = calloc(1, sizeof(BucketTreeContext)); + if (!bktr_ctx) + { + LOG_MSG("Unable to allocate memory for Bucket Tree context! (%u).", storage_type); + goto end; + } + + /* Initialize Bucket Tree context. */ + success = bktrInitializeContext(bktr_ctx, nca_fs_ctx, storage_type); + if (!success) + { + LOG_MSG("Failed to initialize Bucket Tree context! (%u).", storage_type); + goto end; + } + + /* Update output context pointer. */ + *out = bktr_ctx; + +end: + if (!success && bktr_ctx) free(bktr_ctx); + + return success; +} + +NX_INLINE bool ncaStorageIsValidContext(NcaStorageContext *ctx) +{ + return (ctx && ctx->base_storage_type >= NcaStorageBaseStorageType_Regular && ctx->base_storage_type <= NcaStorageBaseStorageType_Compressed && ctx->nca_fs_ctx && \ + ctx->nca_fs_ctx->enabled); +}