[exosphere] Rewrite crt0/init and ld script

exosphere/linker.ld

@@ -3,7 +3,9 @@ ENTRY(__start_cold)
 
 MEMORY
 {
-    fake  : ORIGIN = 0,           LENGTH = 4096M
+    NULL  : ORIGIN = 0,           LENGTH = 4K
+    ccrt0 : ORIGIN = 0x040006000, LENGTH = 4K
+    glob  : ORIGIN = 0x040020000, LENGTH = 128K
     tzram : ORIGIN = 0x07C010000, LENGTH = 64K
     /*
         The warmboot crt0 is preceeded by the exception vector page and the L2 and L3 translation tables.
@@ -21,12 +23,14 @@ MEMORY
 
 SECTIONS
 {
-    PROVIDE(__start__ = 0x0);
+    PROVIDE(__start__ = 0x040006000);
     . = __start__;
 
     .cold_crt0 :
     {
         . = ALIGN(64);
+        __cold_crt0_start__ = ABSOLUTE(.);
+        __glob_origin__ = ORIGIN(glob);
         KEEP (*(.cold_crt0.text.start)) /* MUST be first */
         KEEP (*(.cold_crt0.text*))
         KEEP (build/coldboot_init.o(.text*))
@@ -36,9 +40,10 @@ SECTIONS
         build/coldboot_init.o(.data*)
         . = ALIGN(8);
         *(.cold_crt0.bss*)
-        build/coldboot_init.o(.bss*)
+        build/coldboot_init.o(.bss* COMMON)
         . = ALIGN(64);
-    } >fake AT>fake
+        __cold_crt0_end__ = ABSOLUTE(.);
+    } >ccrt0 AT>glob
 
     .pk2ldr :
     {
@@ -49,16 +54,16 @@ SECTIONS
         build/package2.o(.rodata*)
         build/package2.o(.data*)
         . = ALIGN(8);
-    } >pk2ldr AT>fake
+    } >pk2ldr AT>glob
 
     .pk2ldr.bss :
     {
         . = ALIGN(8);
         __pk2ldr_bss_start__ = ABSOLUTE(.);
-        build/package2.o(.bss*)
+        build/package2.o(.bss* COMMON)
         . = ALIGN(8);
         __pk2ldr_end__ = ABSOLUTE(.);
-    } >pk2ldr AT>fake
+    } >pk2ldr AT>glob
 
     .vectors :
     {
@@ -68,7 +73,7 @@ SECTIONS
         KEEP (*(.vectors*))
         . = ALIGN(8);
         __vectors_end__ = ABSOLUTE(.);
-    } >evt AT>fake
+    } >evt AT>glob
 
     .warm_crt0 :
     {
@@ -87,7 +92,7 @@ SECTIONS
         build/warmboot_init.o(.bss*)
         . = ALIGN(64);
         __warmboot_crt0_end__ = ABSOLUTE(.);
-    } >warmboot_crt0 AT>fake
+    } >warmboot_crt0 AT>glob
 
     .text :
     {
@@ -100,37 +105,37 @@ SECTIONS
         *(.text.hot .text.hot.*)
         *(.text .stub .text.* .gnu.linkonce.t.*)
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
     .init :
     {
         KEEP( *(.init) )
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
     .plt :
     {
         *(.plt)
         *(.iplt)
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
 
     .fini :
     {
         KEEP( *(.fini) )
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
     .rodata :
     {
         *(.rodata .rodata.* .gnu.linkonce.r.*)
         SORT(CONSTRUCTORS)
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
-    .got            : { __got_start__ = ABSOLUTE(.); *(.got) *(.igot) } >main AT>fake
+    .got            : { __got_start__ = ABSOLUTE(.); *(.got) *(.igot) } >main AT>glob
-    .got.plt        : { *(.got.plt)  *(.igot.plt)  __got_end__ = ABSOLUTE(.);} >main AT>fake
+    .got.plt        : { *(.got.plt)  *(.igot.plt)  __got_end__ = ABSOLUTE(.);} >main AT>glob
 
     .preinit_array :
     {
@@ -139,7 +144,7 @@ SECTIONS
         KEEP (*(.preinit_array))
         PROVIDE (__preinit_array_end = .);
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
     .init_array :
     {
@@ -147,7 +152,7 @@ SECTIONS
         KEEP (*(SORT(.init_array.*)))
         KEEP (*(.init_array))
         PROVIDE (__init_array_end = .);
-    } >main AT>fake
+    } >main AT>glob
 
     .fini_array :
     {
@@ -157,7 +162,7 @@ SECTIONS
         KEEP (*(SORT(.fini_array.*)))
         PROVIDE (__fini_array_end = .);
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
     .ctors :
     {
@@ -167,7 +172,7 @@ SECTIONS
         KEEP (*(SORT(.ctors.*)))
         KEEP (*(.ctors))
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
     .dtors ALIGN(8) :
     {
@@ -177,33 +182,33 @@ SECTIONS
         KEEP (*(SORT(.dtors.*)))
         KEEP (*(.dtors))
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
     .data ALIGN(8) :
     {
         *(.data .data.* .gnu.linkonce.d.*)
         CONSTRUCTORS
         . = ALIGN(8);
-    } >main AT>fake
+    } >main AT>glob
 
 
-    .eh_frame_hdr     : { *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*) } >main AT>fake
+    .eh_frame_hdr     : { *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*) } >main AT>glob
-    .eh_frame         : { KEEP (*(.eh_frame)) *(.eh_frame.*) } >main AT>fake
+    .eh_frame         : { KEEP (*(.eh_frame)) *(.eh_frame.*) } >main AT>glob
-    .gcc_except_table : { *(.gcc_except_table .gcc_except_table.*) } >main AT>fake
+    .gcc_except_table : { *(.gcc_except_table .gcc_except_table.*) } >main AT>glob
-    .gnu_extab        : { *(.gnu_extab*) } >main AT>fake
+    .gnu_extab        : { *(.gnu_extab*) } >main AT>glob
-    .exception_ranges : { *(.exception_ranges .exception_ranges*) } >main AT>fake
+    .exception_ranges : { *(.exception_ranges .exception_ranges*) } >main AT>glob
 
-    .dynamic           : { *(.dynamic) } >main AT>fake
+    .dynamic           : { *(.dynamic) } >main AT>glob
-    .interp            : { *(.interp) } >main AT>fake
+    .interp            : { *(.interp) } >main AT>glob
-    .note.gnu.build-id : { *(.note.gnu.build-id) } >main AT>fake
+    .note.gnu.build-id : { *(.note.gnu.build-id) } >main AT>glob
-    .hash              : { *(.hash) } >main AT>fake
+    .hash              : { *(.hash) } >main AT>glob
-    .gnu.hash          : { *(.gnu.hash) } >main AT>fake
+    .gnu.hash          : { *(.gnu.hash) } >main AT>glob
-    .gnu.version       : { *(.gnu.version) } >main AT>fake
+    .gnu.version       : { *(.gnu.version) } >main AT>glob
-    .gnu.version_d     : { *(.gnu.version_d) } >main AT>fake
+    .gnu.version_d     : { *(.gnu.version_d) } >main AT>glob
-    .gnu.version_r     : { *(.gnu.version_r) } >main AT>fake
+    .gnu.version_r     : { *(.gnu.version_r) } >main AT>glob
-    .dynsym            : { *(.dynsym) } >main AT>fake
+    .dynsym            : { *(.dynsym) } >main AT>glob
-    .dynstr            : { *(.dynstr) } >main AT>fake
+    .dynstr            : { *(.dynstr) } >main AT>glob
-    .rela.dyn          : { *(.rela.*); __main_end__ = ABSOLUTE(.);} >main AT>fake
+    .rela.dyn          : { *(.rela.*); __main_end__ = ABSOLUTE(.);} >main AT>glob
 
     .bss :
     {
@@ -215,7 +220,7 @@ SECTIONS
         *(COMMON)
         . = ALIGN(8);
         __main_end__ = ABSOLUTE(.);
-    } >main AT>fake
+    } >main AT>glob
 
     __end__ = ABSOLUTE(.) ;
 
@@ -230,35 +235,35 @@ SECTIONS
     /DISCARD/ : { *(.group .comment .note) }
 
     /* Stabs debugging sections. */
-    .stab          0 : { *(.stab) }
+    .stab          0 : { *(.stab) } AT>glob
-    .stabstr       0 : { *(.stabstr) }
+    .stabstr       0 : { *(.stabstr) } AT>glob
-    .stab.excl     0 : { *(.stab.excl) }
+    .stab.excl     0 : { *(.stab.excl) } AT>glob
-    .stab.exclstr  0 : { *(.stab.exclstr) }
+    .stab.exclstr  0 : { *(.stab.exclstr) } AT>glob
-    .stab.index    0 : { *(.stab.index) }
+    .stab.index    0 : { *(.stab.index) } AT>glob
-    .stab.indexstr 0 : { *(.stab.indexstr) }
+    .stab.indexstr 0 : { *(.stab.indexstr) } AT>glob
 
     /* DWARF debug sections.
        Symbols in the DWARF debugging sections are relative to the beginning
        of the section so we begin them at 0. */
 
     /* DWARF 1 */
-    .debug          0 : { *(.debug) }
+    .debug          0 : { *(.debug) } AT>glob
-    .line           0 : { *(.line) }
+    .line           0 : { *(.line) } AT>glob
 
     /* GNU DWARF 1 extensions */
-    .debug_srcinfo  0 : { *(.debug_srcinfo) }
+    .debug_srcinfo  0 : { *(.debug_srcinfo) } AT>glob
-    .debug_sfnames  0 : { *(.debug_sfnames) }
+    .debug_sfnames  0 : { *(.debug_sfnames) } AT>glob
 
     /* DWARF 1.1 and DWARF 2 */
-    .debug_aranges  0 : { *(.debug_aranges) }
+    .debug_aranges  0 : { *(.debug_aranges) } AT>glob
-    .debug_pubnames 0 : { *(.debug_pubnames) }
+    .debug_pubnames 0 : { *(.debug_pubnames) } AT>glob
 
     /* DWARF 2 */
-    .debug_info     0 : { *(.debug_info) }
+    .debug_info     0 : { *(.debug_info) } AT>glob
-    .debug_abbrev   0 : { *(.debug_abbrev) }
+    .debug_abbrev   0 : { *(.debug_abbrev) } AT>glob
-    .debug_line     0 : { *(.debug_line) }
+    .debug_line     0 : { *(.debug_line) } AT>glob
-    .debug_frame    0 : { *(.debug_frame) }
+    .debug_frame    0 : { *(.debug_frame) } AT>glob
-    .debug_str      0 : { *(.debug_str) }
+    .debug_str      0 : { *(.debug_str) } AT>glob
-    .debug_loc      0 : { *(.debug_loc) }
+    .debug_loc      0 : { *(.debug_loc) } AT>glob
-    .debug_macinfo  0 : { *(.debug_macinfo) }
+    .debug_macinfo  0 : { *(.debug_macinfo) } AT>glob
 }

exosphere/src/arm.s

@@ -63,7 +63,7 @@ tlb_invalidate_page_inner_shareable:
  * x1: 0 clean & invalidate, 1 invalidate only
  * x2~x9: clobbered
  */
-.section    .text.__asm_dcache_level, "ax", %progbits
+.section    .warm_crt0.text.__asm_dcache_level, "ax", %progbits
 .type       __asm_dcache_level, %function
 __asm_dcache_level:
     lsl x12, x0, #1
@@ -108,7 +108,7 @@ loop_way:
  *
  * flush or invalidate all data cache by SET/WAY.
  */
-.section    .text.__asm_dcache_all, "ax", %progbits
+.section    .warm_crt0.text.__asm_dcache_all, "ax", %progbits
 .type       __asm_dcache_all, %function
 __asm_dcache_all:
     mov x1, x0
@@ -146,14 +146,14 @@ skip:
 finished:
     ret
 
-.section    .text.flush_dcache_all, "ax", %progbits
+.section    .warm_crt0.text.flush_dcache_all, "ax", %progbits
 .type       flush_dcache_all, %function
 .global     flush_dcache_all
 flush_dcache_all:
     mov x0, #0
     b   __asm_dcache_all
 
-.section    .text.invalidate_dcache_all, "ax", %progbits
+.section    .warm_crt0.text.invalidate_dcache_all, "ax", %progbits
 .type       invalidate_dcache_all, %function
 .global     invalidate_dcache_all
 invalidate_dcache_all:
@@ -168,7 +168,7 @@ invalidate_dcache_all:
  * x0: start address
  * x1: end address
  */
-.section    .text.flush_dcache_range, "ax", %progbits
+.section    .warm_crt0.text.flush_dcache_range, "ax", %progbits
 .type       flush_dcache_range, %function
 .global     flush_dcache_range
 flush_dcache_range:
@@ -196,7 +196,7 @@ flush_dcache_range:
  * x0: start address
  * x1: end address
  */
-.section    .text.invalidate_dcache_range, "ax", %progbits
+.section    .warm_crt0.text.invalidate_dcache_range, "ax", %progbits
 .type       invalidate_dcache_range, %function
 .global     invalidate_dcache_range
 invalidate_dcache_range:
@@ -220,7 +220,7 @@ invalidate_dcache_range:
  *
  * invalidate all icache entries.
  */
-.section    .text.invalidate_icache_all_inner_shareable, "ax", %progbits
+.section    .warm_crt0.text.invalidate_icache_all_inner_shareable, "ax", %progbits
 .type       invalidate_icache_all_inner_shareable, %function
 .global     invalidate_icache_all_inner_shareable
 invalidate_icache_all_inner_shareable:
@@ -231,7 +231,7 @@ invalidate_icache_all_inner_shareable:
     isb
     ret
 
-.section    .text.invalidate_icache_all, "ax", %progbits
+.section    .warm_crt0.text.invalidate_icache_all, "ax", %progbits
 .type       invalidate_icache_all, %function
 .global     invalidate_icache_all
 invalidate_icache_all:

exosphere/src/bootup.c

@@ -266,17 +266,17 @@ void setup_current_core_state(void) {
 
 void identity_unmap_iram_cd_tzram(void) {
     /* See also: configure_ttbls (in coldboot_init.c). */
-    uintptr_t *mmu_l1_tbl = (uintptr_t *)(TZRAM_GET_SEGMENT_ADDRESS(TZRAM_SEGEMENT_ID_SECMON_EVT) + 0x800 - 64);
+    /*uintptr_t *mmu_l1_tbl = (uintptr_t *)(TZRAM_GET_SEGMENT_ADDRESS(TZRAM_SEGEMENT_ID_SECMON_EVT) + 0x800 - 64);
-    uintptr_t *mmu_l2_tbl = (uintptr_t *)TZRAM_GET_SEGMENT_ADDRESS(TZRAM_SEGMENT_ID_L2_TRANSLATION_TABLE);
+    uintptr_t *mmu_l2_tbl = (uintptr_t *)TZRAM_GET_SEGMENT_ADDRESS(TZRAM_SEGMENT_ID_L2_TRANSLATION_TABLE);*/
     uintptr_t *mmu_l3_tbl = (uintptr_t *)TZRAM_GET_SEGMENT_ADDRESS(TZRAM_SEGMENT_ID_L3_TRANSLATION_TABLE);
 
     mmu_unmap_range(3, mmu_l3_tbl, IDENTITY_GET_MAPPING_ADDRESS(IDENTITY_MAPPING_IRAM_CD), IDENTITY_GET_MAPPING_SIZE(IDENTITY_MAPPING_IRAM_CD));
-    mmu_unmap_range(3, mmu_l3_tbl, IDENTITY_GET_MAPPING_ADDRESS(IDENTITY_MAPPING_TZRAM), IDENTITY_GET_MAPPING_SIZE(IDENTITY_MAPPING_TZRAM));
+    /*mmu_unmap_range(3, mmu_l3_tbl, IDENTITY_GET_MAPPING_ADDRESS(IDENTITY_MAPPING_TZRAM), IDENTITY_GET_MAPPING_SIZE(IDENTITY_MAPPING_TZRAM));
 
     mmu_unmap(2, mmu_l2_tbl, 0x40000000);
     mmu_unmap(2, mmu_l2_tbl, 0x7C000000);
 
-    mmu_unmap(1, mmu_l1_tbl, 0x40000000);
+    mmu_unmap(1, mmu_l1_tbl, 0x40000000);*/
 
     tlb_invalidate_all_inner_shareable();
 }

exosphere/src/coldboot_init.c

@@ -1,5 +1,6 @@
 #include <string.h>
 #include "utils.h"
+#include "arm.h"
 #include "mmu.h"
 #include "memory_map.h"
 #include "arm.h"
@@ -14,6 +15,11 @@
 
 extern const uint8_t __start_cold[];
 
+/* warboot_init.c */
+extern unsigned int g_exosphere_target_firmware_for_init;
+void init_dma_controllers(unsigned int target_firmware);
+void set_memory_registers_enable_mmu(void);
+
 static void identity_map_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t *mmu_l3_tbl) {
     static const uintptr_t addrs[]      =   { TUPLE_FOLD_LEFT_0(EVAL(IDENTIY_MAPPING_ID_MAX), _MMAPID, COMMA) };
     static const size_t sizes[]         =   { TUPLE_FOLD_LEFT_1(EVAL(IDENTIY_MAPPING_ID_MAX), _MMAPID, COMMA) };
@@ -98,32 +104,15 @@ static void configure_ttbls(void) {
     tzram_map_all_segments(mmu_l3_tbl);
 }
 
-static void translate_func_list(coldboot_crt0_reloc_list_t *reloc_list, boot_func_list_t *func_list, bool in_tzram) {
-    coldboot_crt0_reloc_t *warmboot_crt0_reloc = &reloc_list->relocs[0];
-    coldboot_crt0_reloc_t *main_reloc = &reloc_list->relocs[reloc_list->nb_relocs_pre_mmu_init];
-
-    uintptr_t main_pa;
-    if (in_tzram) {
-        /* The main segment immediately follows the warmboot crt0 in TZRAM, in the same page. */
-        main_pa = (uintptr_t)warmboot_crt0_reloc->vma | ((uintptr_t)main_reloc->vma & 0xFFF);
-    } else {
-        main_pa = reloc_list->reloc_base + main_reloc->reloc_offset;
-    }
-
-    for(size_t i = 0; i < func_list->nb_funcs; i++) {
-        if(func_list->addrs[i] >= 0x1F0000000ull) {
-            func_list->addrs[i] = main_pa + func_list->addrs[i] - (uintptr_t)main_reloc->vma;
-        }
-    }
-}
-
 static void do_relocation(const coldboot_crt0_reloc_list_t *reloc_list, size_t index) {
+    extern const uint8_t __glob_origin__[];
     uint64_t *p_vma = (uint64_t *)reloc_list->relocs[index].vma;
-    const uint64_t *p_lma = (const uint64_t *)(reloc_list->reloc_base + reloc_list->relocs[index].reloc_offset);
+    size_t offset = reloc_list->relocs[index].lma - (uintptr_t)__glob_origin__;
+    const uint64_t *p_lma = (const uint64_t *)(reloc_list->reloc_base + offset);
     size_t size = reloc_list->relocs[index].end_vma - reloc_list->relocs[index].vma;
 
     for(size_t i = 0; i < size / 8; i++) {
-        p_vma[i] = reloc_list->relocs[index].reloc_offset != 0 ? p_lma[i] : 0;
+        p_vma[i] = offset != 0 ? p_lma[i] : 0;
     }
 }
 
@@ -131,14 +120,19 @@ uintptr_t get_coldboot_crt0_stack_address(void) {
     return TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_CORE3_STACK) + 0x800;
 }
 
-void coldboot_init(coldboot_crt0_reloc_list_t *reloc_list, boot_func_list_t *func_list, boot_func_list_t *func_list_warmboot, uintptr_t start_cold) {
+void coldboot_init(coldboot_crt0_reloc_list_t *reloc_list, uintptr_t start_cold) {
     //MAILBOX_NX_SECMON_BOOT_TIME = TIMERUS_CNTR_1US_0;
 
-    boot_func_list_t func_copy = *func_list;
     /* Custom approach */
     reloc_list->reloc_base = start_cold;
-    translate_func_list(reloc_list, func_list, false);
 
+    /* TODO: Set NX BOOTLOADER clock time field */
+
+    /* This at least copies .warm_crt0 to its VMA. */
+    for(size_t i = 0; i < reloc_list->nb_relocs_pre_mmu_init; i++) {
+        do_relocation(reloc_list, i);
+    }
+    /* At this point, we can (and will) access functions located in .warm_crt0 */
 
     /*
         From https://events.static.linuxfound.org/sites/events/files/slides/slides_17.pdf :
@@ -149,37 +143,25 @@ void coldboot_init(coldboot_crt0_reloc_list_t *reloc_list, boot_func_list_t *fun
 
         It should be fine to clear that here and not before.
     */
-    func_list->funcs.flush_dcache_all();
+    flush_dcache_all();
-    func_list->funcs.invalidate_icache_all();
+    invalidate_icache_all();
-    
-
-    /* TODO: Set NX BOOTLOADER clock time field */
-    
-    /* This at least copies .warm_crt0 to its VMA. */
-    for(size_t i = 0; i < reloc_list->nb_relocs_pre_mmu_init; i++) {
-        do_relocation(reloc_list, i);
-    }
-    /* At this point, we can (and will) access functions located in .warm_crt0 */
 
     /* Set target firmware. */
-    func_list->target_firmware = exosphere_get_target_firmware_physical();
+    g_exosphere_target_firmware_for_init = exosphere_get_target_firmware_for_init();
 
     /* Initialize DMA controllers, and write to AHB_GIZMO_TZRAM. */
     /* TZRAM accesses should work normally after this point. */
-    func_list->funcs.init_dma_controllers(func_list->target_firmware);
+    init_dma_controllers(g_exosphere_target_firmware_for_init);
 
     configure_ttbls();
-    func_list->funcs.set_memory_registers_enable_mmu();
+    set_memory_registers_enable_mmu();
 
     /* Copy or clear the remaining sections */
     for(size_t i = 0; i < reloc_list->nb_relocs_post_mmu_init; i++) {
         do_relocation(reloc_list, reloc_list->nb_relocs_pre_mmu_init + i);
     }
 
-    func_list->funcs.flush_dcache_all();
+    flush_dcache_all();
-    func_list->funcs.invalidate_icache_all();
+    invalidate_icache_all();
     /* At this point we can access all the mapped segments (all other functions, data...) normally */
-
-    *func_list_warmboot = func_copy;
-    translate_func_list(reloc_list, func_list_warmboot, true);
 }

exosphere/src/cpu_context.c

@@ -23,9 +23,6 @@
 /* start.s */
 void __attribute__((noreturn)) __jump_to_lower_el(uint64_t arg, uintptr_t ep, uint32_t spsr);
 
-/* See notes in start.s */
-critical_section_t g_boot_critical_section = {{{.ticket_number = 1}}};
-
 static saved_cpu_context_t g_cpu_contexts[NUM_CPU_CORES] = {0};
 
 void use_core_entrypoint_and_argument(uint32_t core, uintptr_t *entrypoint_addr, uint64_t *argument) {
@@ -47,15 +44,22 @@ void set_core_entrypoint_and_argument(uint32_t core, uintptr_t entrypoint_addr,
     g_cpu_contexts[core].argument = argument;
 }
 
+static __attribute__((target("cmodel=large"), noinline))
+critical_section_t *get_boot_critical_section(void) {
+    return &g_boot_critical_section;
+}
+
 void __attribute__((noreturn)) core_jump_to_lower_el(void) {
     uintptr_t ep;
     uint64_t arg;
     unsigned int core_id = get_core_id();
     uint32_t spsr = get_spsr();
+    critical_section_t *critsec = get_boot_critical_section();
 
     use_core_entrypoint_and_argument(core_id, &ep, &arg);
-    critical_section_leave(&g_boot_critical_section);
+    critical_section_leave(critsec);
-    flush_dcache_range(&g_boot_critical_section, (uint8_t *)&g_boot_critical_section + sizeof(g_boot_critical_section)); /* already does a dsb sy */
+    flush_dcache_range(critsec, (uint8_t *)critsec + sizeof(critical_section_t));
+    /* already does a dsb sy */
     __sev();
 
     /* Nintendo hardcodes EL1, but we can boot fine using other EL1/EL2 modes as well */

exosphere/src/exocfg.h

@@ -34,7 +34,7 @@ typedef struct {
 unsigned int exosphere_load_config(void);
 unsigned int exosphere_get_target_firmware(void);
 
-static inline unsigned int exosphere_get_target_firmware_physical(void) {
+static inline unsigned int exosphere_get_target_firmware_for_init(void) {
     return MAILBOX_EXOSPHERE_CONFIG_PHYS.magic == MAGIC_EXOSPHERE_BOOTCONFIG ? MAILBOX_EXOSPHERE_CONFIG_PHYS.target_firmware : EXOSPHERE_TARGET_FIRMWARE_DEFAULT_FOR_DEBUG;
 }
 

exosphere/src/package2.c

@@ -438,6 +438,7 @@ void load_package2(coldboot_crt0_reloc_list_t *reloc_list) {
     secure_additional_devices();
 
     /* Remove the identity mapping for iRAM-C+D & TZRAM */
+    /* For our crt0 to work, this doesn't actually unmap TZRAM */
     identity_unmap_iram_cd_tzram();
 
     /* Load header from NX_BOOTLOADER-initialized DRAM. */

exosphere/src/start.s

@@ -81,6 +81,24 @@ __start_cold:
         However we'll initialize g_boot_critical_section so that it acts like core0 has entered it,
         for it to be in .data and for safety.
     */
+
+    /* Relocate the crt0. Nintendo doesn't do it. */
+    ldr  x0, =__cold_crt0_start__
+    adr  x1, __start_cold
+    ldr  x2, =__cold_crt0_end__
+    cmp  x0, x1
+    beq  _post_cold_crt0_reloc
+    1:
+        ldp  x3, x4, [x0], #0x10
+        stp  x3, x4, [x1], #0x10
+        cmp  x0, x2
+        blo  1b
+
+    ldr  x16, =_post_cold_crt0_reloc
+    br   x16
+
+_post_cold_crt0_reloc:
+
     msr  spsel, #0
     bl   get_coldboot_crt0_stack_address
     mov  sp, x0
@@ -88,9 +106,7 @@ __start_cold:
 
     adr  x0, g_coldboot_crt0_relocation_list
     mov  x19, x0
-    adr  x1, g_coldboot_crt0_main_func_list
+    adr  x1, __start_cold
-    ldr  x2, =g_warmboot_crt0_main_func_list
-    adr  x3, __start_cold
     bl   coldboot_init
 
     ldr  x16, =__jump_to_main_cold
@@ -112,17 +128,12 @@ __start_warm:
 
     /* PA(__main_start__) = __warmboot_crt0_start__ + 0x800 (refer to the linker script) */
     ldr  x0, =g_boot_critical_section
-    ldr  x1, =__main_start__
-    sub  x0, x0, x1
-    ldr  x1, =(__start_warm + 0x800)
-    add  x0, x0, x1
     bl   warmboot_crt0_critical_section_enter
 
     bl   get_warmboot_crt0_stack_address
     mov  sp, x0
     mov  fp, #0
 
-    adr  x0, g_warmboot_crt0_main_func_list
     bl   warmboot_init
     ldr  x16, =__jump_to_main_warm
     br   x16
@@ -238,20 +249,9 @@ g_coldboot_crt0_relocation_list:
     .quad   __main_bss_start__, __main_end__, 0
     .quad   __pk2ldr_bss_start__, __pk2ldr_end__, 0
 
-.section    .cold_crt0.data.g_coldboot_crt0_main_func_list, "aw", %progbits
+/* Critical section */
-.align      3
+.section    .warm_crt0.data.g_boot_critical_section, "aw", %progbits
-.global     g_coldboot_crt0_main_func_list
+.align      2
-g_coldboot_crt0_main_func_list:
+.global     g_boot_critical_section
-    .quad   4   /* Number of functions */
+g_boot_critical_section:
-    .quad   0   /* Target firmware, overwritten in coldboot_init. */
+    .word   1 /* Core0 entered, by default. */
-    /* Functions */
-    .quad   init_dma_controllers
-    .quad   set_memory_registers_enable_mmu
-    .quad   flush_dcache_all
-    .quad   invalidate_icache_all
-
-.section    .warm_crt0.data.g_warmboot_crt0_main_func_list, "aw", %progbits
-.align      3
-.global     g_warmboot_crt0_main_func_list
-g_warmboot_crt0_main_func_list:
-    .space (6 * 8)

exosphere/src/utils.h

@@ -25,26 +25,11 @@
 
 /* Custom stuff below */
 
-/* For warmboot (and coldboot crt0) */
-typedef struct {
-    size_t      nb_funcs;
-    uint64_t target_firmware;
-    union {
-        struct {
-            void (*init_dma_controllers)(unsigned int);
-            void (*set_memory_registers_enable_mmu)(void);
-            void (*flush_dcache_all)(void);
-            void (*invalidate_icache_all)(void);
-        } funcs;
-        uintptr_t addrs[4];
-    };
-} boot_func_list_t;
-
 /* For coldboot */
 typedef struct {
     uint8_t     *vma;
     uint8_t     *end_vma;
-    uintptr_t   reloc_offset;
+    uintptr_t   lma;
 } coldboot_crt0_reloc_t;
 
 typedef struct {

exosphere/src/warmboot_init.c

@@ -12,6 +12,8 @@
 void __set_memory_registers(uintptr_t ttbr0, uintptr_t vbar, uint64_t cpuectlr, uint32_t scr,
                             uint32_t tcr, uint32_t cptr, uint64_t mair, uint32_t sctlr);
 
+unsigned int g_exosphere_target_firmware_for_init = 0;
+
 uintptr_t get_warmboot_crt0_stack_address(void) {
     return TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_CORE012_STACK) + 0x800;
 }
@@ -140,6 +142,7 @@ void set_memory_registers_enable_mmu(void) {
     __set_memory_registers(ttbr0, vbar, cpuectlr, scr, tcr, cptr, mair, sctlr);
 }
 
+#if 0 /* Since we decided not to identity-unmap TZRAM */
 static void identity_remap_tzram(void) {
     /* See also: configure_ttbls (in coldboot_init.c). */
     uintptr_t *mmu_l1_tbl = (uintptr_t *)(TZRAM_GET_SEGMENT_PA(TZRAM_SEGEMENT_ID_SECMON_EVT) + 0x800 - 64);
@@ -153,8 +156,9 @@ static void identity_remap_tzram(void) {
                          IDENTITY_GET_MAPPING_SIZE(IDENTITY_MAPPING_TZRAM), IDENTITY_GET_MAPPING_ATTRIBS(IDENTITY_MAPPING_TZRAM),
                          IDENTITY_IS_MAPPING_BLOCK_RANGE(IDENTITY_MAPPING_TZRAM));
 }
+#endif
 
-void warmboot_init(boot_func_list_t *func_list) {
+void warmboot_init(void) {
     /*
         From https://events.static.linuxfound.org/sites/events/files/slides/slides_17.pdf :
         Caches may write back dirty lines at any time:
@@ -162,15 +166,15 @@ void warmboot_init(boot_func_list_t *func_list) {
             - Even if MMU is off
             - Even if Cacheable accesses are disabled (caches are never 'off')
     */
-    func_list->funcs.flush_dcache_all();
+    flush_dcache_all();
-    func_list->funcs.invalidate_icache_all();
+    invalidate_icache_all();
 
     /* On warmboot (not cpu_on) only */
     if (MC_SECURITY_CFG3_0 == 0) {
-        init_dma_controllers(func_list->target_firmware);
+        init_dma_controllers(g_exosphere_target_firmware_for_init);
     }
 
-    identity_remap_tzram();
+    /*identity_remap_tzram();*/
     /* Nintendo pointlessly fully invalidate the TLB & invalidate the data cache on the modified ranges here */
     set_memory_registers_enable_mmu();
 }

exosphere/src/warmboot_main.c

@@ -27,6 +27,7 @@ void __attribute__((noreturn)) warmboot_main(void) {
     }
 
     /* IRAM C+D identity mapping has actually been removed on coldboot but we don't really care */
+    /* For our crt0 to work, this doesn't actually unmap TZRAM */
     identity_unmap_iram_cd_tzram();
 
     /* On warmboot (not cpu_on) only */
@@ -60,7 +61,8 @@ void __attribute__((noreturn)) warmboot_main(void) {
         clear_user_smc_in_progress();
 
         if (exosphere_get_target_firmware() >= EXOSPHERE_TARGET_FIRMWARE_400) {
-            setup_4x_mmio();        }
+            setup_4x_mmio();
+        }
     }
 
     setup_current_core_state();