Implement layered NCA storage interface.

Updating PFS and RomFS comes next. Also fixed Sparse storages not being supported as an indirect storage's original substorage.

include/core/bktr.h

@@ -154,10 +154,10 @@ typedef enum {
                                                 ///< May be used as substorage for all BucketTreeStorage types.
     BucketTreeSubStorageType_Indirect   = 1,    ///< Indirect storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Compressed only.
     BucketTreeSubStorageType_AesCtrEx   = 2,    ///< AesCtrEx storage from patch NCAs. May be used as substorage for BucketTreeStorageType_Indirect only.
-    BucketTreeSubStorageType_Sparse     = 3,    ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Used in base applications only.
+    BucketTreeSubStorageType_Compressed = 3,    ///< Compressed storage. If available, this is always the outmost storage type for any NCA. May be used by all title types.
-                                                ///< May be used as substorage for BucketTreeStorageType_Compressed only.
-    BucketTreeSubStorageType_Compressed = 4,    ///< Compressed storage. If available, this is always the outmost storage type for any NCA. May be used by all title types.
                                                 ///< May be used as substorage for BucketTreeStorageType_Indirect only.
+    BucketTreeSubStorageType_Sparse     = 4,    ///< Sparse storage with CTR crypto, using virtual offsets as lower CTR IVs. Used in base applications only.
+                                                ///< May be used as substorage for BucketTreeStorageType_Compressed or BucketTreeStorageType_Indirect.
     BucketTreeSubStorageType_Count      = 5     ///< Total values supported by this enum.
 } BucketTreeSubStorageType;
 

include/core/nca_storage.h

@@ -0,0 +1,68 @@
+/*
+ * nca_storage.h
+ *
+ * Copyright (c) 2020-2022, DarkMatterCore <pabloacurielz@gmail.com>.
+ *
+ * This file is part of nxdumptool (https://github.com/DarkMatterCore/nxdumptool).
+ *
+ * nxdumptool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nxdumptool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#ifndef __NCA_STORAGE_H__
+#define __NCA_STORAGE_H__
+
+#include "bktr.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+    NcaStorageBaseStorageType_Invalid    = 0,   /* Placeholder. */
+    NcaStorageBaseStorageType_Regular    = 1,
+    NcaStorageBaseStorageType_Sparse     = 2,
+    NcaStorageBaseStorageType_Indirect   = 3,
+    NcaStorageBaseStorageType_Compressed = 4
+} NcaStorageBaseStorageType;
+
+/// Used to perform multi-layer reads within a single NCA FS section.
+typedef struct {
+    u8 base_storage_type;                   ///< NcaStorageBaseStorageType.
+    NcaFsSectionContext *nca_fs_ctx;        ///< NCA FS section context used to initialize this context.
+    BucketTreeContext *sparse_storage;      ///< Sparse storage context.
+    BucketTreeContext *aes_ctr_ex_storage;  ///< AesCtrEx storage context.
+    BucketTreeContext *indirect_storage;    ///< Indirect storage context.
+    BucketTreeContext *compressed_storage;  ///< Compressed storage context.
+} NcaStorageContext;
+
+/// Initializes a NCA storage context using a NCA FS section context.
+bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx);
+
+/// Sets a storage from the provided Base NcaStorageContext as the original substorage for the provided Patch NcaStorageContext's Indirect Storage.
+/// Needed to perform combined reads between a base NCA and a patch NCA.
+bool ncaStorageSetPatchOriginalSubStorage(NcaStorageContext *patch_ctx, NcaStorageContext *base_ctx);
+
+/// Reads data from the NCA storage using a previously initialized NcaStorageContext.
+bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset);
+
+/// Frees a previously initialized NCA storage context.
+void ncaStorageFreeContext(NcaStorageContext *ctx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __NCA_STORAGE_H__ */

source/core/bktr.c

@@ -198,13 +198,14 @@ bool bktrSetRegularSubStorage(BucketTreeContext *ctx, NcaFsSectionContext *nca_f
 bool bktrSetBucketTreeSubStorage(BucketTreeContext *parent_ctx, BucketTreeContext *child_ctx, u8 substorage_index)
 {
     if (!bktrIsValidContext(parent_ctx) || !bktrIsValidContext(child_ctx) || substorage_index >= BKTR_MAX_SUBSTORAGE_COUNT || \
-        (parent_ctx->storage_type != BucketTreeStorageType_Indirect && substorage_index != 0) || \
-        (parent_ctx->storage_type == BucketTreeStorageType_Indirect && child_ctx->storage_type != BucketTreeStorageType_Compressed && child_ctx->storage_type != BucketTreeStorageType_AesCtrEx) || \
-        (parent_ctx->storage_type == BucketTreeStorageType_Indirect && child_ctx->storage_type == BucketTreeStorageType_Compressed && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx)) || \
-        (parent_ctx->storage_type == BucketTreeStorageType_Indirect && child_ctx->storage_type == BucketTreeStorageType_AesCtrEx && (substorage_index != 1 || parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) || \
         parent_ctx->storage_type == BucketTreeStorageType_AesCtrEx || parent_ctx->storage_type == BucketTreeStorageType_Sparse || \
-        (parent_ctx->storage_type == BucketTreeStorageType_Compressed && child_ctx->storage_type != BucketTreeStorageType_Indirect && child_ctx->storage_type != BucketTreeStorageType_Sparse) || \
+        (parent_ctx->storage_type != BucketTreeStorageType_Indirect && substorage_index != 0) || \
-        (parent_ctx->storage_type == BucketTreeStorageType_Compressed && parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx))
+        (parent_ctx->storage_type == BucketTreeStorageType_Indirect && (child_ctx->storage_type < BucketTreeStorageType_AesCtrEx || \
+        child_ctx->storage_type > BucketTreeStorageType_Sparse || (child_ctx->storage_type == BucketTreeStorageType_AesCtrEx && (substorage_index != 1 || \
+        parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)) || ((child_ctx->storage_type == BucketTreeStorageType_Compressed || \
+        child_ctx->storage_type == BucketTreeStorageType_Sparse) && (substorage_index != 0 || parent_ctx->nca_fs_ctx == child_ctx->nca_fs_ctx)))) || \
+        (parent_ctx->storage_type == BucketTreeStorageType_Compressed && ((child_ctx->storage_type != BucketTreeStorageType_Indirect && \
+        child_ctx->storage_type != BucketTreeStorageType_Sparse) || parent_ctx->nca_fs_ctx != child_ctx->nca_fs_ctx)))
     {
         LOG_MSG("Invalid parameters!");
         return false;
@@ -364,7 +365,8 @@ static bool bktrReadIndirectStorage(BucketTreeVisitor *visitor, void *out, u64 r
     bool is_sparse = (ctx->storage_type == BucketTreeStorageType_Sparse);
     
     if (!out || !bktrIsValidSubstorage(&(ctx->substorages[0])) || (!is_sparse && !bktrIsValidSubstorage(&(ctx->substorages[1]))) || \
-        (!is_sparse && ((ctx->substorages[0].type != BucketTreeSubStorageType_Regular && ctx->substorages[0].type != BucketTreeStorageType_Compressed) || ctx->substorages[1].type != BucketTreeSubStorageType_AesCtrEx)) || \
+        (!is_sparse && ((ctx->substorages[0].type != BucketTreeSubStorageType_Regular && ctx->substorages[0].type != BucketTreeStorageType_Compressed && \
+        ctx->substorages[0].type != BucketTreeSubStorageType_Sparse) || ctx->substorages[1].type != BucketTreeSubStorageType_AesCtrEx)) || \
         (is_sparse && ctx->substorages[0].type != BucketTreeSubStorageType_Regular))
     {
         LOG_MSG("Invalid parameters!");

source/core/nca_storage.c

@@ -0,0 +1,252 @@
+/*
+ * nca_storage.c
+ *
+ * Copyright (c) 2020-2022, DarkMatterCore <pabloacurielz@gmail.com>.
+ *
+ * This file is part of nxdumptool (https://github.com/DarkMatterCore/nxdumptool).
+ *
+ * nxdumptool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nxdumptool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "nxdt_utils.h"
+#include "nca_storage.h"
+
+/* Function prototypes. */
+
+static bool ncaStorageInitializeBucketTreeContext(BucketTreeContext **out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type);
+NX_INLINE bool ncaStorageIsValidContext(NcaStorageContext *ctx);
+
+bool ncaStorageInitializeContext(NcaStorageContext *out, NcaFsSectionContext *nca_fs_ctx)
+{
+    /* TODO: allow patches with sparse layers? */
+    if (!out || !nca_fs_ctx || !nca_fs_ctx->enabled || (nca_fs_ctx->has_sparse_layer && nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs))
+    {
+        LOG_MSG("Invalid parameters!");
+        return false;
+    }
+    
+    bool success = false;
+    
+    /* Free output context beforehand. */
+    ncaStorageFreeContext(out);
+    
+    /* Set initial base storage type. */
+    out->base_storage_type = NcaStorageBaseStorageType_Regular;
+    
+    /* Check if a sparse layer is available. */
+    if (nca_fs_ctx->has_sparse_layer)
+    {
+        /* Initialize sparse layer. */
+        if (!ncaStorageInitializeBucketTreeContext(&(out->sparse_storage), nca_fs_ctx, BucketTreeStorageType_Sparse)) goto end;
+        
+        /* Set sparse layer's substorage. */
+        if (!bktrSetRegularSubStorage(out->sparse_storage, nca_fs_ctx)) goto end;
+        
+        /* Update base storage type. */
+        out->base_storage_type = NcaStorageBaseStorageType_Sparse;
+    }
+    
+    /* Check if both Indirect and AesCtrEx layers are available. */
+    if (nca_fs_ctx->section_type == NcaFsSectionType_PatchRomFs)
+    {
+        /* Initialize AesCtrEx and Indirect layers. */
+        if (!ncaStorageInitializeBucketTreeContext(&(out->aes_ctr_ex_storage), nca_fs_ctx, BucketTreeStorageType_AesCtrEx) || \
+            !ncaStorageInitializeBucketTreeContext(&(out->indirect_storage), nca_fs_ctx, BucketTreeStorageType_Indirect)) goto end;
+        
+        /* Set AesCtrEx layer's substorage. */
+        if (!bktrSetRegularSubStorage(out->aes_ctr_ex_storage, nca_fs_ctx)) goto end;
+        
+        /* Set Indirect layer's AesCtrEx substorage. */
+        /* Base substorage must be manually set at a later time. */
+        if (!bktrSetBucketTreeSubStorage(out->indirect_storage, out->aes_ctr_ex_storage, 1)) goto end;
+        
+        /* Update base storage type. */
+        out->base_storage_type = NcaStorageBaseStorageType_Indirect;
+    }
+    
+    /* Check if a compression layer is available. */
+    if (nca_fs_ctx->has_compression_layer)
+    {
+        /* Initialize compression layer. */
+        if (!ncaStorageInitializeBucketTreeContext(&(out->compressed_storage), nca_fs_ctx, BucketTreeStorageType_Compressed)) goto end;
+        
+        /* Set compression layer's substorage. */
+        switch(out->base_storage_type)
+        {
+            case NcaStorageBaseStorageType_Regular:
+                if (!bktrSetRegularSubStorage(out->compressed_storage, nca_fs_ctx)) goto end;
+                break;
+            case NcaStorageBaseStorageType_Sparse:
+                if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->sparse_storage, 0)) goto end;
+                break;
+            case NcaStorageBaseStorageType_Indirect:
+                if (!bktrSetBucketTreeSubStorage(out->compressed_storage, out->indirect_storage, 0)) goto end;
+                break;
+        }
+        
+        /* Update base storage type. */
+        out->base_storage_type = NcaStorageBaseStorageType_Compressed;
+    }
+    
+    /* Update return value. */
+    success = true;
+    
+end:
+    if (!success) ncaStorageFreeContext(out);
+    
+    return success;
+}
+
+bool ncaStorageSetPatchOriginalSubStorage(NcaStorageContext *patch_ctx, NcaStorageContext *base_ctx)
+{
+    NcaContext *patch_nca_ctx = NULL, *base_nca_ctx = NULL;
+    
+    if (!ncaStorageIsValidContext(patch_ctx) || !ncaStorageIsValidContext(base_ctx) || patch_ctx->nca_fs_ctx == base_ctx->nca_fs_ctx || \
+        !(patch_nca_ctx = (NcaContext*)patch_ctx->nca_fs_ctx->nca_ctx) || !(base_nca_ctx = (NcaContext*)base_ctx->nca_fs_ctx->nca_ctx) || \
+        patch_ctx->nca_fs_ctx->section_type != NcaFsSectionType_PatchRomFs || base_ctx->nca_fs_ctx->section_type != NcaFsSectionType_RomFs || \
+        patch_nca_ctx->header.program_id != base_nca_ctx->header.program_id || patch_nca_ctx->header.content_type != base_nca_ctx->header.content_type || \
+        patch_nca_ctx->id_offset != base_nca_ctx->id_offset || patch_nca_ctx->title_version <= base_nca_ctx->title_version || !patch_ctx->indirect_storage)
+    {
+        LOG_MSG("Invalid parameters!");
+        return false;
+    }
+    
+    bool success = false;
+    
+    /* Set base storage. */
+    switch(base_ctx->base_storage_type)
+    {
+        case NcaStorageBaseStorageType_Regular:
+            success = bktrSetRegularSubStorage(patch_ctx->indirect_storage, base_ctx->nca_fs_ctx);
+            break;
+        case NcaStorageBaseStorageType_Sparse:
+            success = bktrSetBucketTreeSubStorage(patch_ctx->indirect_storage, base_ctx->sparse_storage, 0);
+            break;
+        case NcaStorageBaseStorageType_Compressed:
+            success = bktrSetBucketTreeSubStorage(patch_ctx->indirect_storage, base_ctx->compressed_storage, 0);
+            break;
+        default:
+            break;
+    }
+    
+    if (!success) LOG_MSG("Failed to set base storage to patch storage!");
+    
+    return success;
+}
+
+bool ncaStorageRead(NcaStorageContext *ctx, void *out, u64 read_size, u64 offset)
+{
+    if (!ncaStorageIsValidContext(ctx) || !out || !read_size)
+    {
+        LOG_MSG("Invalid parameters!");
+        return false;
+    }
+    
+    bool success = false;
+    
+    switch(ctx->base_storage_type)
+    {
+        case NcaStorageBaseStorageType_Regular:
+            success = ncaReadFsSection(ctx->nca_fs_ctx, out, read_size, offset);
+            break;
+        case NcaStorageBaseStorageType_Sparse:
+            success = bktrReadStorage(ctx->sparse_storage, out, read_size, offset);
+            break;
+        case NcaStorageBaseStorageType_Indirect:
+            success = bktrReadStorage(ctx->indirect_storage, out, read_size, offset);
+            break;
+        case NcaStorageBaseStorageType_Compressed:
+            success = bktrReadStorage(ctx->compressed_storage, out, read_size, offset);
+            break;
+        default:
+            break;
+    }
+    
+    if (!success) LOG_MSG("Failed to read 0x%lX-byte long block from offset 0x%lX in base storage! (%u).", read_size, offset, ctx->base_storage_type);
+    
+    return success;
+}
+
+void ncaStorageFreeContext(NcaStorageContext *ctx)
+{
+    if (!ctx) return;
+    
+    if (ctx->sparse_storage)
+    {
+        bktrFreeContext(ctx->sparse_storage);
+        free(ctx->sparse_storage);
+    }
+    
+    if (ctx->aes_ctr_ex_storage)
+    {
+        bktrFreeContext(ctx->aes_ctr_ex_storage);
+        free(ctx->aes_ctr_ex_storage);
+    }
+    
+    if (ctx->indirect_storage)
+    {
+        bktrFreeContext(ctx->indirect_storage);
+        free(ctx->indirect_storage);
+    }
+    
+    if (ctx->compressed_storage)
+    {
+        bktrFreeContext(ctx->compressed_storage);
+        free(ctx->compressed_storage);
+    }
+    
+    memset(ctx, 0, sizeof(NcaStorageContext));
+}
+
+static bool ncaStorageInitializeBucketTreeContext(BucketTreeContext **out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type)
+{
+    if (!out || !nca_fs_ctx || storage_type >= BucketTreeStorageType_Count)
+    {
+        LOG_MSG("Invalid parameters!");
+        return false;
+    }
+    
+    BucketTreeContext *bktr_ctx = NULL;
+    bool success = false;
+    
+    /* Allocate memory for the Bucket Tree context. */
+    bktr_ctx = calloc(1, sizeof(BucketTreeContext));
+    if (!bktr_ctx)
+    {
+        LOG_MSG("Unable to allocate memory for Bucket Tree context! (%u).", storage_type);
+        goto end;
+    }
+    
+    /* Initialize Bucket Tree context. */
+    success = bktrInitializeContext(bktr_ctx, nca_fs_ctx, storage_type);
+    if (!success)
+    {
+        LOG_MSG("Failed to initialize Bucket Tree context! (%u).", storage_type);
+        goto end;
+    }
+    
+    /* Update output context pointer. */
+    *out = bktr_ctx;
+    
+end:
+    if (!success && bktr_ctx) free(bktr_ctx);
+    
+    return success;
+}
+
+NX_INLINE bool ncaStorageIsValidContext(NcaStorageContext *ctx)
+{
+    return (ctx && ctx->base_storage_type >= NcaStorageBaseStorageType_Regular && ctx->base_storage_type <= NcaStorageBaseStorageType_Compressed && ctx->nca_fs_ctx && \
+            ctx->nca_fs_ctx->enabled);
+}