BKTR rewrite: part 1.

This is a mess. It won't build, so don't bother trying.

include/core/bktr.h

@@ -31,67 +31,184 @@
 extern "C" {
 #endif
 
+#define BKTR_NODE_HEADER_SIZE               0x10
+#define BKTR_NODE_SIZE                      0x4000                      /* Currently shared by all Bucket Tree storage types. */
+#define BKTR_NODE_SIZE_MIN                  0x400
+#define BKTR_NODE_SIZE_MAX                  0x80000
+
+#define BKTR_INDIRECT_ENTRY_SIZE            0x14
+#define BKTR_AES_CTR_EX_ENTRY_SIZE          0x10
+#define BKTR_COMPRESSED_ENTRY_SIZE          0x18
+
+#define BKTR_COMPRESSION_PHYS_ALIGNMENT     0x10
+
+#define BKTR_COMPRESSION_LEVEL_MIN          0
+#define BKTR_COMPRESSION_LEVEL_MAX          16
+#define BKTR_COMPRESSION_LEVEL_DEFAULT      BKTR_COMPRESSION_LEVEL_MIN
+
+#define BKTR_COMPRESSION_INVALID_PHYS_SIZE  UINT32_MAX
+
+/// Used as the header for both BucketTreeOffsetNode and BucketTreeEntryNode.
+typedef struct {
+    u32 index;  ///< BucketTreeOffsetNode / BucketTreeEntryNode index.
+    u32 count;  ///< BucketTreeHeader: BucketTreeEntryNode count. BucketTreeEntryNode: entry count.
+    u64 offset; ///< Usually represents a physical or virtual size.
+} BucketTreeNodeHeader;
+
+NXDT_ASSERT(BucketTreeNodeHeader, BKTR_NODE_HEADER_SIZE);
+
+/// First segment of every BucketTreeTable.
+typedef struct {
+    BucketTreeNodeHeader header;
+    u64 offsets[0x7FE];             ///< May represent virtual or physical offsets, depending on the storage type.
+} BucketTreeOffsetNode;
+
+NXDT_ASSERT(BucketTreeOffsetNode, BKTR_NODE_SIZE);
+
+/// IndirectStorage-related elements.
 typedef enum {
-    BktrIndirectStorageIndex_Original = 0,
+    BucketTreeIndirectStorageIndex_Original = 0,
-    BktrIndirectStorageIndex_Patch    = 1
+    BucketTreeIndirectStorageIndex_Patch    = 1
-} BktrIndirectStorageIndex;
+} BucketTreeIndirectStorageIndex;
 
 #pragma pack(push, 1)
 typedef struct {
     u64 virtual_offset;
     u64 physical_offset;
-    u32 indirect_storage_index; ///< BktrIndirectStorageIndex.
+    u32 storage_index;      ///< BucketTreeIndirectStorageIndex.
-} BktrIndirectStorageEntry;
+} BucketTreeIndirectStorageEntry;
 #pragma pack(pop)
 
-NXDT_ASSERT(BktrIndirectStorageEntry, 0x14);
+NXDT_ASSERT(BucketTreeIndirectStorageEntry, BKTR_INDIRECT_ENTRY_SIZE);
 
-typedef struct {
+/// AesCtrExStorage-related elements.
-    u32 index;
+typedef enum {
-    u32 entry_count;
+    BucketTreeAesCtrExStorageEncryption_Enabled  = 0,
-    u64 end_offset;
+    BucketTreeAesCtrExStorageEncryption_Disabled = 1
-    BktrIndirectStorageEntry indirect_storage_entries[0x3FF0 / sizeof(BktrIndirectStorageEntry)];
+} BucketTreeAesCtrExStorageEncryption;
-    u8 reserved[0x3FF0 % sizeof(BktrIndirectStorageEntry)];
-} BktrIndirectStorageBucket;
-
-NXDT_ASSERT(BktrIndirectStorageBucket, 0x4000);
-
-typedef struct {
-    u32 index;
-    u32 bucket_count;
-    u64 virtual_size;
-    u64 virtual_offsets[0x3FF0 / sizeof(u64)];
-    BktrIndirectStorageBucket indirect_storage_buckets[];
-} BktrIndirectStorageBlock;
-
-NXDT_ASSERT(BktrIndirectStorageBlock, 0x4000);
 
 typedef struct {
     u64 offset;
-    u32 size;
+    u8 encryption;      ///< BucketTreeAesCtrExStorageEncryption.
+    u8 reserved[0x3];
     u32 generation;
-} BktrAesCtrExStorageEntry;
+} BucketTreeAesCtrExStorageEntry;
 
-NXDT_ASSERT(BktrAesCtrExStorageEntry, 0x10);
+NXDT_ASSERT(BucketTreeAesCtrExStorageEntry, BKTR_AES_CTR_EX_ENTRY_SIZE);
+
+/// CompressedStorage-related elements.
+typedef enum {
+    BucketTreeCompressedStorageCompressionType_None = 0,
+    BucketTreeCompressedStorageCompressionType_Zero = 1,
+    BucketTreeCompressedStorageCompressionType_2    = 2,
+    BucketTreeCompressedStorageCompressionType_LZ4  = 3
+} BucketTreeCompressedStorageCompressionType;
 
 typedef struct {
-    u32 index;
+    s64 virtual_offset;
-    u32 entry_count;
+    s64 physical_offset;    ///< Must be aligned to BKTR_COMPRESSION_PHYS_ALIGNMENT.
-    u64 end_offset;
+    u8 compression_type;    ///< BucketTreeCompressedStorageCompressionType.
-    BktrAesCtrExStorageEntry aes_ctr_ex_storage_entries[0x3FF];
+    s8 compression_level;   ///< Must be within the range [BKTR_COMPRESSION_LEVEL_MIN, BKTR_COMPRESSION_LEVEL_MAX].
-} BktrAesCtrExStorageBucket;
+    u8 reserved[0x2];
+    u32 physical_size;
+} BucketTreeCompressedStorageEntry;
 
-NXDT_ASSERT(BktrAesCtrExStorageBucket, 0x4000);
+NXDT_ASSERT(BucketTreeCompressedStorageEntry, BKTR_COMPRESSED_ENTRY_SIZE);
+
+/// Second segment of every BucketTreeTable. At least one entry node must be available.
+typedef struct {
+    BucketTreeNodeHeader header;
+    union {
+        struct {
+            BucketTreeIndirectStorageEntry indirect_entries[0x332];
+            u8 reserved[0x8];
+        };
+        BucketTreeAesCtrExStorageEntry aes_ctr_ex_entries[0x3FF];
+        BucketTreeCompressedStorageEntry compressed_entries[0x2AA];
+    };
+} BucketTreeEntryNode;
+
+NXDT_ASSERT(BucketTreeEntryNode, BKTR_NODE_SIZE);
 
 typedef struct {
-    u32 index;
+    BucketTreeOffsetNode offset_node;
-    u32 bucket_count;
+    BucketTreeEntryNode entry_nodes[];  ///< Number of nodes can be retrieved from offset_node.header.count.
-    u64 physical_size;
+} BucketTreeTable;
-    u64 physical_offsets[0x3FF0 / sizeof(u64)];
+
-    BktrAesCtrExStorageBucket aes_ctr_ex_storage_buckets[];
+NXDT_ASSERT(BucketTreeTable, BKTR_NODE_SIZE);
-} BktrAesCtrExStorageBlock;
+
+typedef enum {
+    BucketTreeStorageType_Indirect   = 0,
+    BucketTreeStorageType_AesCtrEx   = 1,
+    BucketTreeStorageType_Compressed = 2,
+    BucketTreeStorageType_Sparse     = 3,   ///< BucketTreeStorageType_Indirect, but uses virtual offsets for data decryption.
+    BucketTreeStorageType_Count      = 4    ///< Total values supported by this enum.
+} BucketTreeStorageType;
+
+typedef struct {
+    NcaFsSectionContext *nca_fs_ctx;    ///< NCA FS section context. Used to perform operations on the target NCA.
+    NcaBucketInfo bucket;               ///< Bucket info used to initialize this context.
+    u8 storage_type;                    ///< BucketTreeStorageType.
+    BucketTreeTable *storage_table;     ///< Pointer to the dynamically allocated Bucket Tree Table for this storage.
+    u64 node_size;                      ///< Node size for this type of Bucket Tree storage.
+    u64 entry_size;                     ///< Size of each individual entry within BucketTreeEntryNode.
+    u32 offset_count;                   ///< Number of offsets available within each BucketTreeOffsetNode for this storage.
+    u32 entry_set_count;                ///< Number of BucketTreeEntryNode elements available in this storage.
+    u64 start_offset;                   ///< Virtual storage start offset.
+    u64 end_offset;                     ///< Virtual storage end offset.
+} BucketTreeContext;
+
+/// Initializes a Bucket Tree context using the provided NCA FS section context and a storage 
+bool bktrInitializeContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type);
+
+/// Reads data from a Bucket Tree storage using a previously initialized BucketTreeContext.
+bool bktrReadStorage(BucketTreeContext *ctx, void *out, u64 read_size, u64 offset);
+
+/// Helper inline functions.
+
+NX_INLINE void bktrFreeContext(BucketTreeContext *ctx)
+{
+    if (!ctx) return;
+    if (ctx->storage_table) free(ctx->storage_table);
+    memset(ctx, 0, sizeof(BucketTreeContext));
+}
+
+NX_INLINE bool bktrIsValidContext(BucketTreeContext *ctx)
+{
+    return (ctx && ctx->nca_fs_ctx && ctx->storage_type < BucketTreeStorageType_Count && ctx->storage_table && ctx->node_size && ctx->entry_size && ctx->offset_count && \
+            ctx->entry_set_count && ctx->end_offset > ctx->start_offset);
+}
+
+NX_INLINE bool bktrIsOffsetWithinStorageRange(BucketTreeContext *ctx, u64 offset)
+{
+    return (bktrIsValidContext(ctx) && ctx->start_offset <= offset && offset < ctx->end_offset);
+}
+
+NX_INLINE bool bktrIsBlockWithinStorageRange(BucketTreeContext *ctx, u64 size, u64 offset)
+{
+    return (bktrIsValidContext(ctx) && size > 0 && ctx->start_offset <= offset && size <= (ctx->end_offset - offset));
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-NXDT_ASSERT(BktrAesCtrExStorageBlock, 0x4000);
 
 typedef struct {
     RomFileSystemContext base_romfs_ctx;            ///< Base NCA RomFS context.

include/core/nca.h

@@ -372,7 +372,9 @@ typedef struct {
     u8 hash_type;                       ///< NcaHashType.
     u8 encryption_type;                 ///< NcaEncryptionType.
     u8 section_type;                    ///< NcaFsSectionType.
-    bool skip_hash_layer_crypto;        ///< Set to true if hash layer decryption should be skipped while reading section data.
+    
+    ///< Hash-layer-related fields.
+    bool skip_hash_layer_crypto;        ///< Set to true if hash layer crypto should be skipped while reading section data.
     NcaRegion hash_region;              ///< Holds the properties for the full hash layer region that precedes the actual FS section data.
     
     ///< Crypto-related fields.
@@ -384,19 +386,10 @@ typedef struct {
     ///< SparseInfo-related fields.
     bool has_sparse_layer;              ///< Set to true if this NCA FS section has a sparse layer.
     u64 sparse_table_offset;            ///< header.sparse_info.physical_offset + header.sparse_info.bucket.offset. Relative to the start of the NCA content file. Placed here for convenience.
-    u64 sparse_table_size;              ///< header.sparse_info.bucket.size. Placed here for convenience.
-    u8 sparse_ctr[AES_BLOCK_SIZE];      ///< AES-128-CTR IV used for sparse table decryption.
-    Aes128CtrContext sparse_ctr_ctx;    ///< AES-128-CTR context used for sparse table decryption.
-    u64 cur_sparse_virtual_offset;      ///< Current sparse layer virtual offset. Used for content decryption if a sparse layer is available.
     
     ///< CompressionInfo-related fields.
     bool has_compression_layer;         ///< Set to true if this NCA FS section has a compression layer.
-    u64 compression_table_offset;       ///< section_offset + hash_target_offset + header.compression_info.bucket.offset. Relative to the start of the NCA content file. Placed here for convenience.
+    u64 compression_table_offset;       ///< hash_target_offset + header.compression_info.bucket.offset. Relative to the start of the FS section. Placed here for convenience.
-    u64 compression_table_size;         ///< header.compression_info.bucket.size. Placed here for convenience.
-    
-    
-    
-    
     
     ///< NSP-related fields.
     bool header_written;                ///< Set to true after this FS section header has been written to an output dump.
@@ -560,6 +553,11 @@ NX_INLINE bool ncaIsHeaderDirty(NcaContext *ctx)
     return (memcmp(tmp_hash, ctx->header_hash, SHA256_HASH_SIZE) != 0);
 }
 
+NX_INLINE bool ncaVerifyBucketInfo(NcaBucketInfo *bucket)
+{
+    return (bucket && __builtin_bswap32(bucket->header.magic) == NCA_BKTR_MAGIC && bucket->header.version <= NCA_BKTR_VERSION && bucket->header.entry_count >= 0);
+}
+
 NX_INLINE void ncaFreeHierarchicalSha256Patch(NcaHierarchicalSha256Patch *patch)
 {
     if (!patch) return;

include/defines.h

@@ -39,7 +39,9 @@
 #define ALIGN_DOWN(x, y)                ((x) & ~((y) - 1))
 #define IS_ALIGNED(x, y)                (((x) & ((y) - 1)) == 0)
 
-#define IS_POWER_OF_TWO(x)              (((x) & ((x) - 1)) == 0)
+#define IS_POWER_OF_TWO(x)              ((x) > 0 && ((x) & ((x) - 1)) == 0)
+
+#define DIVIDE_UP(x, y)                 (((x) + ((y) - 1)) / (y))
 
 #define CONCATENATE_IMPL(s1, s2)        s1##s2
 #define CONCATENATE(s1, s2)             CONCATENATE_IMPL(s1, s2)

source/core/nacp.c

@@ -1012,7 +1012,7 @@ const char *nacpGetContentsAvailabilityTransitionPolicyString(u8 contents_availa
 
 NX_INLINE bool nacpCheckBitflagField(const void *flag, u8 flag_bitcount, u8 idx)
 {
-    if (!flag || !flag_bitcount || !IS_POWER_OF_TWO(flag_bitcount) || idx >= flag_bitcount) return false;
+    if (!flag || !IS_POWER_OF_TWO(flag_bitcount) || idx >= flag_bitcount) return false;
     const u8 *flag_u8 = (const u8*)flag;
     u8 byte_idx = (idx >> 3);
     u8 bitmask = BIT(idx - ALIGN_DOWN(idx, 8));

source/core/nca.c

@@ -836,8 +836,7 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
         u64 raw_storage_offset = sparse_info->physical_offset;
         u64 raw_storage_size = (sparse_bucket->offset + sparse_bucket->size);
         
-        if (__builtin_bswap32(sparse_bucket->header.magic) != NCA_BKTR_MAGIC || sparse_bucket->header.version != NCA_BKTR_VERSION || raw_storage_offset < sizeof(NcaHeader) || \
+        if (!ncaVerifyBucketInfo(sparse_bucket) || raw_storage_offset < sizeof(NcaHeader) || (raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
-            (raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
         {
             LOG_DATA(sparse_info, sizeof(NcaSparseInfo), "Invalid SparseInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. SparseInfo dump:", section_idx, \
                      nca_ctx->content_id_str, nca_ctx->content_size);
@@ -852,11 +851,8 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
             goto end;
         }
         
-        /* Set sparse table properties. */
+        /* Update context. */
         fs_ctx->sparse_table_offset = (sparse_info->physical_offset + sparse_bucket->offset);
-        fs_ctx->sparse_table_size = sparse_bucket->size;
-        
-        /* Update section size. */
         fs_ctx->section_size = raw_storage_size;
     }
     
@@ -877,18 +873,16 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
         raw_storage_offset += compression_bucket->offset;
         
         /* Check if the compression bucket is valid. */
-        if (__builtin_bswap32(compression_bucket->header.magic) != NCA_BKTR_MAGIC || compression_bucket->header.version != NCA_BKTR_VERSION || !sparse_bucket->header.entry_count || \
+        if (!ncaVerifyBucketInfo(compression_bucket) || !compression_bucket->header.entry_count || raw_storage_offset < sizeof(NcaHeader) || \
-            raw_storage_offset < sizeof(NcaHeader) || (raw_storage_offset + raw_storage_size) > fs_ctx->section_size || \
+            (raw_storage_offset + raw_storage_size) > fs_ctx->section_size || (fs_ctx->section_offset + raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
-            (fs_ctx->section_offset + raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
         {
-            LOG_DATA(sparse_info, sizeof(NcaSparseInfo), "Invalid CompressionInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. CompressionInfo dump:", section_idx, \
+            LOG_DATA(compression_bucket, sizeof(NcaBucketInfo), "Invalid CompressionInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. CompressionInfo dump:", \
-                     nca_ctx->content_id_str, nca_ctx->content_size);
+                     section_idx, nca_ctx->content_id_str, nca_ctx->content_size);
             goto end;
         }
         
-        /* Set sparse table properties. */
+        /* Update context. */
-        fs_ctx->compression_table_offset = (fs_ctx->section_offset + raw_storage_offset);
+        fs_ctx->compression_table_offset = raw_storage_offset;
-        fs_ctx->compression_table_size = raw_storage_size;
     }
     
     /* Check if we're within boundaries. */
@@ -980,21 +974,11 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
         /* Initialize the partial AES counter for this section. */
         aes128CtrInitializePartialCtr(fs_ctx->ctr, fs_ctx->header.aes_ctr_upper_iv.value, fs_ctx->section_offset);
         
-        if (fs_ctx->has_sparse_layer)
-        {
-            /* Initialize the partial AES counter for the sparse info bucket table. */
-            NcaAesCtrUpperIv sparse_upper_iv = {0};
-            memcpy(sparse_upper_iv.value, fs_ctx->header.aes_ctr_upper_iv.value, sizeof(sparse_upper_iv.value));
-            sparse_upper_iv.generation = ((u32)(sparse_info->generation) << 16);
-            aes128CtrInitializePartialCtr(fs_ctx->sparse_ctr, sparse_upper_iv.value, fs_ctx->sparse_table_offset);
-        }
-        
         /* Initialize AES context. */
         if (nca_ctx->rights_id_available)
         {
             /* AES-128-CTR is always used for FS crypto in NCAs with a rights ID. */
             aes128CtrContextCreate(&(fs_ctx->ctr_ctx), nca_ctx->titlekey, fs_ctx->ctr);
-            if (fs_ctx->has_sparse_layer) aes128CtrContextCreate(&(fs_ctx->sparse_ctr_ctx), nca_ctx->titlekey, fs_ctx->sparse_ctr);
         } else {
             if (fs_ctx->encryption_type == NcaEncryptionType_AesXts)
             {
@@ -1006,7 +990,6 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
             {
                 /* Patch RomFS sections also use the AES-128-CTR key from the decrypted NCA key area, for some reason. */
                 aes128CtrContextCreate(&(fs_ctx->ctr_ctx), nca_ctx->decrypted_key_area.aes_ctr, fs_ctx->ctr);
-                if (fs_ctx->has_sparse_layer) aes128CtrContextCreate(&(fs_ctx->sparse_ctr_ctx), nca_ctx->decrypted_key_area.aes_ctr, fs_ctx->sparse_ctr);
             }
         }
     }