mirrors/nxdumptool
1
0
Fork 0
mirror of https://github.com/DarkMatterCore/nxdumptool.git synced 2024-11-22 10:16:39 +00:00
Code Issues Releases Activity

BKTR rewrite: part 1.

Browse source 
This is a mess. It won't build, so don't bother trying.
This commit is contained in:
Pablo Curiel 2022-07-02 12:09:49 +02:00
parent e372b97131
commit 29e5af2064
6 changed files with 946 additions and 609 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

199
include/core/bktr.h
View file

@ -31,67 +31,184 @@
extern "C" {
#endif
#define BKTR_NODE_HEADER_SIZE 0x10
#define BKTR_NODE_SIZE 0x4000 /* Currently shared by all Bucket Tree storage types. */
#define BKTR_NODE_SIZE_MIN 0x400
#define BKTR_NODE_SIZE_MAX 0x80000
#define BKTR_INDIRECT_ENTRY_SIZE 0x14
#define BKTR_AES_CTR_EX_ENTRY_SIZE 0x10
#define BKTR_COMPRESSED_ENTRY_SIZE 0x18
#define BKTR_COMPRESSION_PHYS_ALIGNMENT 0x10
#define BKTR_COMPRESSION_LEVEL_MIN 0
#define BKTR_COMPRESSION_LEVEL_MAX 16
#define BKTR_COMPRESSION_LEVEL_DEFAULT BKTR_COMPRESSION_LEVEL_MIN
#define BKTR_COMPRESSION_INVALID_PHYS_SIZE UINT32_MAX
/// Used as the header for both BucketTreeOffsetNode and BucketTreeEntryNode.
typedef struct {
u32 index; ///< BucketTreeOffsetNode / BucketTreeEntryNode index.
u32 count; ///< BucketTreeHeader: BucketTreeEntryNode count. BucketTreeEntryNode: entry count.
u64 offset; ///< Usually represents a physical or virtual size.
} BucketTreeNodeHeader;
NXDT_ASSERT(BucketTreeNodeHeader, BKTR_NODE_HEADER_SIZE);
/// First segment of every BucketTreeTable.
typedef struct {
BucketTreeNodeHeader header;
u64 offsets[0x7FE]; ///< May represent virtual or physical offsets, depending on the storage type.
} BucketTreeOffsetNode;
NXDT_ASSERT(BucketTreeOffsetNode, BKTR_NODE_SIZE);
/// IndirectStorage-related elements.
typedef enum {
BktrIndirectStorageIndex_Original = 0,
BktrIndirectStorageIndex_Patch = 1
} BktrIndirectStorageIndex;
BucketTreeIndirectStorageIndex_Original = 0,
BucketTreeIndirectStorageIndex_Patch = 1
} BucketTreeIndirectStorageIndex;
#pragma pack(push, 1)
typedef struct {
u64 virtual_offset;
u64 physical_offset;
u32 indirect_storage_index; ///< BktrIndirectStorageIndex.
} BktrIndirectStorageEntry;
u32 storage_index; ///< BucketTreeIndirectStorageIndex.
} BucketTreeIndirectStorageEntry;
#pragma pack(pop)
NXDT_ASSERT(BktrIndirectStorageEntry, 0x14);
NXDT_ASSERT(BucketTreeIndirectStorageEntry, BKTR_INDIRECT_ENTRY_SIZE);
typedef struct {
u32 index;
u32 entry_count;
u64 end_offset;
BktrIndirectStorageEntry indirect_storage_entries[0x3FF0 / sizeof(BktrIndirectStorageEntry)];
u8 reserved[0x3FF0 % sizeof(BktrIndirectStorageEntry)];
} BktrIndirectStorageBucket;
NXDT_ASSERT(BktrIndirectStorageBucket, 0x4000);
typedef struct {
u32 index;
u32 bucket_count;
u64 virtual_size;
u64 virtual_offsets[0x3FF0 / sizeof(u64)];
BktrIndirectStorageBucket indirect_storage_buckets[];
} BktrIndirectStorageBlock;
NXDT_ASSERT(BktrIndirectStorageBlock, 0x4000);
/// AesCtrExStorage-related elements.
typedef enum {
BucketTreeAesCtrExStorageEncryption_Enabled = 0,
BucketTreeAesCtrExStorageEncryption_Disabled = 1
} BucketTreeAesCtrExStorageEncryption;
typedef struct {
u64 offset;
u32 size;
u8 encryption; ///< BucketTreeAesCtrExStorageEncryption.
u8 reserved[0x3];
u32 generation;
} BktrAesCtrExStorageEntry;
} BucketTreeAesCtrExStorageEntry;
NXDT_ASSERT(BktrAesCtrExStorageEntry, 0x10);
NXDT_ASSERT(BucketTreeAesCtrExStorageEntry, BKTR_AES_CTR_EX_ENTRY_SIZE);
/// CompressedStorage-related elements.
typedef enum {
BucketTreeCompressedStorageCompressionType_None = 0,
BucketTreeCompressedStorageCompressionType_Zero = 1,
BucketTreeCompressedStorageCompressionType_2 = 2,
BucketTreeCompressedStorageCompressionType_LZ4 = 3
} BucketTreeCompressedStorageCompressionType;
typedef struct {
u32 index;
u32 entry_count;
u64 end_offset;
BktrAesCtrExStorageEntry aes_ctr_ex_storage_entries[0x3FF];
} BktrAesCtrExStorageBucket;
s64 virtual_offset;
s64 physical_offset; ///< Must be aligned to BKTR_COMPRESSION_PHYS_ALIGNMENT.
u8 compression_type; ///< BucketTreeCompressedStorageCompressionType.
s8 compression_level; ///< Must be within the range [BKTR_COMPRESSION_LEVEL_MIN, BKTR_COMPRESSION_LEVEL_MAX].
u8 reserved[0x2];
u32 physical_size;
} BucketTreeCompressedStorageEntry;
NXDT_ASSERT(BktrAesCtrExStorageBucket, 0x4000);
NXDT_ASSERT(BucketTreeCompressedStorageEntry, BKTR_COMPRESSED_ENTRY_SIZE);
/// Second segment of every BucketTreeTable. At least one entry node must be available.
typedef struct {
BucketTreeNodeHeader header;
union {
struct {
BucketTreeIndirectStorageEntry indirect_entries[0x332];
u8 reserved[0x8];
};
BucketTreeAesCtrExStorageEntry aes_ctr_ex_entries[0x3FF];
BucketTreeCompressedStorageEntry compressed_entries[0x2AA];
};
} BucketTreeEntryNode;
NXDT_ASSERT(BucketTreeEntryNode, BKTR_NODE_SIZE);
typedef struct {
u32 index;
u32 bucket_count;
u64 physical_size;
u64 physical_offsets[0x3FF0 / sizeof(u64)];
BktrAesCtrExStorageBucket aes_ctr_ex_storage_buckets[];
} BktrAesCtrExStorageBlock;
BucketTreeOffsetNode offset_node;
BucketTreeEntryNode entry_nodes[]; ///< Number of nodes can be retrieved from offset_node.header.count.
} BucketTreeTable;
NXDT_ASSERT(BucketTreeTable, BKTR_NODE_SIZE);
typedef enum {
BucketTreeStorageType_Indirect = 0,
BucketTreeStorageType_AesCtrEx = 1,
BucketTreeStorageType_Compressed = 2,
BucketTreeStorageType_Sparse = 3, ///< BucketTreeStorageType_Indirect, but uses virtual offsets for data decryption.
BucketTreeStorageType_Count = 4 ///< Total values supported by this enum.
} BucketTreeStorageType;
typedef struct {
NcaFsSectionContext *nca_fs_ctx; ///< NCA FS section context. Used to perform operations on the target NCA.
NcaBucketInfo bucket; ///< Bucket info used to initialize this context.
u8 storage_type; ///< BucketTreeStorageType.
BucketTreeTable *storage_table; ///< Pointer to the dynamically allocated Bucket Tree Table for this storage.
u64 node_size; ///< Node size for this type of Bucket Tree storage.
u64 entry_size; ///< Size of each individual entry within BucketTreeEntryNode.
u32 offset_count; ///< Number of offsets available within each BucketTreeOffsetNode for this storage.
u32 entry_set_count; ///< Number of BucketTreeEntryNode elements available in this storage.
u64 start_offset; ///< Virtual storage start offset.
u64 end_offset; ///< Virtual storage end offset.
} BucketTreeContext;
/// Initializes a Bucket Tree context using the provided NCA FS section context and a storage
bool bktrInitializeContext(BucketTreeContext *out, NcaFsSectionContext *nca_fs_ctx, u8 storage_type);
/// Reads data from a Bucket Tree storage using a previously initialized BucketTreeContext.
bool bktrReadStorage(BucketTreeContext *ctx, void *out, u64 read_size, u64 offset);
/// Helper inline functions.
NX_INLINE void bktrFreeContext(BucketTreeContext *ctx)
{
if (!ctx) return;
if (ctx->storage_table) free(ctx->storage_table);
memset(ctx, 0, sizeof(BucketTreeContext));
}
NX_INLINE bool bktrIsValidContext(BucketTreeContext *ctx)
{
return (ctx && ctx->nca_fs_ctx && ctx->storage_type < BucketTreeStorageType_Count && ctx->storage_table && ctx->node_size && ctx->entry_size && ctx->offset_count && \
ctx->entry_set_count && ctx->end_offset > ctx->start_offset);
}
NX_INLINE bool bktrIsOffsetWithinStorageRange(BucketTreeContext *ctx, u64 offset)
{
return (bktrIsValidContext(ctx) && ctx->start_offset <= offset && offset < ctx->end_offset);
}
NX_INLINE bool bktrIsBlockWithinStorageRange(BucketTreeContext *ctx, u64 size, u64 offset)
{
return (bktrIsValidContext(ctx) && size > 0 && ctx->start_offset <= offset && size <= (ctx->end_offset - offset));
}
NXDT_ASSERT(BktrAesCtrExStorageBlock, 0x4000);
typedef struct {
RomFileSystemContext base_romfs_ctx; ///< Base NCA RomFS context.

20
include/core/nca.h
View file

@ -372,7 +372,9 @@ typedef struct {
u8 hash_type; ///< NcaHashType.
u8 encryption_type; ///< NcaEncryptionType.
u8 section_type; ///< NcaFsSectionType.
bool skip_hash_layer_crypto; ///< Set to true if hash layer decryption should be skipped while reading section data.
///< Hash-layer-related fields.
bool skip_hash_layer_crypto; ///< Set to true if hash layer crypto should be skipped while reading section data.
NcaRegion hash_region; ///< Holds the properties for the full hash layer region that precedes the actual FS section data.
///< Crypto-related fields.
@ -384,19 +386,10 @@ typedef struct {
///< SparseInfo-related fields.
bool has_sparse_layer; ///< Set to true if this NCA FS section has a sparse layer.
u64 sparse_table_offset; ///< header.sparse_info.physical_offset + header.sparse_info.bucket.offset. Relative to the start of the NCA content file. Placed here for convenience.
u64 sparse_table_size; ///< header.sparse_info.bucket.size. Placed here for convenience.
u8 sparse_ctr[AES_BLOCK_SIZE]; ///< AES-128-CTR IV used for sparse table decryption.
Aes128CtrContext sparse_ctr_ctx; ///< AES-128-CTR context used for sparse table decryption.
u64 cur_sparse_virtual_offset; ///< Current sparse layer virtual offset. Used for content decryption if a sparse layer is available.
///< CompressionInfo-related fields.
bool has_compression_layer; ///< Set to true if this NCA FS section has a compression layer.
u64 compression_table_offset; ///< section_offset + hash_target_offset + header.compression_info.bucket.offset. Relative to the start of the NCA content file. Placed here for convenience.
u64 compression_table_size; ///< header.compression_info.bucket.size. Placed here for convenience.
u64 compression_table_offset; ///< hash_target_offset + header.compression_info.bucket.offset. Relative to the start of the FS section. Placed here for convenience.
///< NSP-related fields.
bool header_written; ///< Set to true after this FS section header has been written to an output dump.
@ -560,6 +553,11 @@ NX_INLINE bool ncaIsHeaderDirty(NcaContext *ctx)
return (memcmp(tmp_hash, ctx->header_hash, SHA256_HASH_SIZE) != 0);
}
NX_INLINE bool ncaVerifyBucketInfo(NcaBucketInfo *bucket)
{
return (bucket && __builtin_bswap32(bucket->header.magic) == NCA_BKTR_MAGIC && bucket->header.version <= NCA_BKTR_VERSION && bucket->header.entry_count >= 0);
}
NX_INLINE void ncaFreeHierarchicalSha256Patch(NcaHierarchicalSha256Patch *patch)
{
if (!patch) return;

4
include/defines.h
View file

@ -39,7 +39,9 @@
#define ALIGN_DOWN(x, y) ((x) & ~((y) - 1))
#define IS_ALIGNED(x, y) (((x) & ((y) - 1)) == 0)
#define IS_POWER_OF_TWO(x) (((x) & ((x) - 1)) == 0)
#define IS_POWER_OF_TWO(x) ((x) > 0 && ((x) & ((x) - 1)) == 0)
#define DIVIDE_UP(x, y) (((x) + ((y) - 1)) / (y))
#define CONCATENATE_IMPL(s1, s2) s1##s2
#define CONCATENATE(s1, s2) CONCATENATE_IMPL(s1, s2)

1297
source/core/bktr.c
View file

File diff suppressed because it is too large Load diff

2
source/core/nacp.c
View file

@ -1012,7 +1012,7 @@ const char *nacpGetContentsAvailabilityTransitionPolicyString(u8 contents_availa
NX_INLINE bool nacpCheckBitflagField(const void *flag, u8 flag_bitcount, u8 idx)
{
if (!flag || !flag_bitcount || !IS_POWER_OF_TWO(flag_bitcount) || idx >= flag_bitcount) return false;
if (!flag || !IS_POWER_OF_TWO(flag_bitcount) || idx >= flag_bitcount) return false;
const u8 *flag_u8 = (const u8*)flag;
u8 byte_idx = (idx >> 3);
u8 bitmask = BIT(idx - ALIGN_DOWN(idx, 8));

33
source/core/nca.c
View file

@ -836,8 +836,7 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
u64 raw_storage_offset = sparse_info->physical_offset;
u64 raw_storage_size = (sparse_bucket->offset + sparse_bucket->size);
if (__builtin_bswap32(sparse_bucket->header.magic) != NCA_BKTR_MAGIC || sparse_bucket->header.version != NCA_BKTR_VERSION || raw_storage_offset < sizeof(NcaHeader) || \
(raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
if (!ncaVerifyBucketInfo(sparse_bucket) || raw_storage_offset < sizeof(NcaHeader) || (raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
{
LOG_DATA(sparse_info, sizeof(NcaSparseInfo), "Invalid SparseInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. SparseInfo dump:", section_idx, \
nca_ctx->content_id_str, nca_ctx->content_size);
@ -852,11 +851,8 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
goto end;
}
/* Set sparse table properties. */
/* Update context. */
fs_ctx->sparse_table_offset = (sparse_info->physical_offset + sparse_bucket->offset);
fs_ctx->sparse_table_size = sparse_bucket->size;
/* Update section size. */
fs_ctx->section_size = raw_storage_size;
}
@ -877,18 +873,16 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
raw_storage_offset += compression_bucket->offset;
/* Check if the compression bucket is valid. */
if (__builtin_bswap32(compression_bucket->header.magic) != NCA_BKTR_MAGIC || compression_bucket->header.version != NCA_BKTR_VERSION || !sparse_bucket->header.entry_count || \
raw_storage_offset < sizeof(NcaHeader) || (raw_storage_offset + raw_storage_size) > fs_ctx->section_size || \
(fs_ctx->section_offset + raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
if (!ncaVerifyBucketInfo(compression_bucket) || !compression_bucket->header.entry_count || raw_storage_offset < sizeof(NcaHeader) || \
(raw_storage_offset + raw_storage_size) > fs_ctx->section_size || (fs_ctx->section_offset + raw_storage_offset + raw_storage_size) > nca_ctx->content_size)
{
LOG_DATA(sparse_info, sizeof(NcaSparseInfo), "Invalid CompressionInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. CompressionInfo dump:", section_idx, \
nca_ctx->content_id_str, nca_ctx->content_size);
LOG_DATA(compression_bucket, sizeof(NcaBucketInfo), "Invalid CompressionInfo data for FS section #%u in \"%s\" (0x%lX). Skipping FS section. CompressionInfo dump:", \
section_idx, nca_ctx->content_id_str, nca_ctx->content_size);
goto end;
}
/* Set sparse table properties. */
fs_ctx->compression_table_offset = (fs_ctx->section_offset + raw_storage_offset);
fs_ctx->compression_table_size = raw_storage_size;
/* Update context. */
fs_ctx->compression_table_offset = raw_storage_offset;
}
/* Check if we're within boundaries. */
@ -980,21 +974,11 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
/* Initialize the partial AES counter for this section. */
aes128CtrInitializePartialCtr(fs_ctx->ctr, fs_ctx->header.aes_ctr_upper_iv.value, fs_ctx->section_offset);
if (fs_ctx->has_sparse_layer)
{
/* Initialize the partial AES counter for the sparse info bucket table. */
NcaAesCtrUpperIv sparse_upper_iv = {0};
memcpy(sparse_upper_iv.value, fs_ctx->header.aes_ctr_upper_iv.value, sizeof(sparse_upper_iv.value));
sparse_upper_iv.generation = ((u32)(sparse_info->generation) << 16);
aes128CtrInitializePartialCtr(fs_ctx->sparse_ctr, sparse_upper_iv.value, fs_ctx->sparse_table_offset);
}
/* Initialize AES context. */
if (nca_ctx->rights_id_available)
{
/* AES-128-CTR is always used for FS crypto in NCAs with a rights ID. */
aes128CtrContextCreate(&(fs_ctx->ctr_ctx), nca_ctx->titlekey, fs_ctx->ctr);
if (fs_ctx->has_sparse_layer) aes128CtrContextCreate(&(fs_ctx->sparse_ctr_ctx), nca_ctx->titlekey, fs_ctx->sparse_ctr);
} else {
if (fs_ctx->encryption_type == NcaEncryptionType_AesXts)
{
@ -1006,7 +990,6 @@ static bool ncaInitializeFsSectionContext(NcaContext *nca_ctx, u32 section_idx)
{
/* Patch RomFS sections also use the AES-128-CTR key from the decrypted NCA key area, for some reason. */
aes128CtrContextCreate(&(fs_ctx->ctr_ctx), nca_ctx->decrypted_key_area.aes_ctr, fs_ctx->ctr);
if (fs_ctx->has_sparse_layer) aes128CtrContextCreate(&(fs_ctx->sparse_ctr_ctx), nca_ctx->decrypted_key_area.aes_ctr, fs_ctx->sparse_ctr);
}
}
}