Implement flush_dcache_all_tzram_pa and invalidate_icache_all_inner_shareable_tzram_pa for the crt0s

exosphere/src/arm.h

@@ -15,7 +15,8 @@ void invalidate_dcache_all(void);
 void flush_dcache_range(const void *start, const void *end);
 void invalidate_dcache_range(const void *start, const void *end);
 
-void invalidate_icache_inner_shareable(void);
+void invalidate_icache_all_inner_shareable(void);
+void invalidate_icache_all(void);
 
 void finalize_powerdown(void);
 void call_with_stack_pointer(uintptr_t stack_pointer, void (*function)(void));

exosphere/src/arm.s

@@ -217,17 +217,28 @@ invalidate_dcache_range:
  *
  * invalidate all icache entries.
  */
-.section    .text.invalidate_icache_inner_shareable, "ax", %progbits
+.section    .text.invalidate_icache_all_inner_shareable, "ax", %progbits
-.type       invalidate_icache_inner_shareable, %function
+.type       invalidate_icache_all_inner_shareable, %function
-.global     invalidate_icache_inner_shareable
+.global     invalidate_icache_all_inner_shareable
-invalidate_icache_inner_shareable:
+invalidate_icache_all_inner_shareable:
     dsb ish
     isb
     ic  ialluis
     dsb ish
     isb
     ret
-    
+
+.section    .text.invalidate_icache_all, "ax", %progbits
+.type       invalidate_icache_all, %function
+.global     invalidate_icache_all
+invalidate_icache_all:
+    dsb sy
+    isb
+    ic  iallu
+    dsb sy
+    isb
+    ret
+
 /* Final steps before power down. */
 .section    .text.finalize_powerdown, "ax", %progbits
 .type       finalize_powerdown, %function
@@ -255,7 +266,7 @@ finalize_powerdown:
     /* Disable receiving instruction cache/tbl maintenance operations. */
     mrs x0, s3_1_c15_c2_1
     and x0, x0, #0xffffffffffffffbf
-    msr s3_1_c15_c2_1, x0 
+    msr s3_1_c15_c2_1, x0
     /* Prepare GICC */
     bl intr_prepare_gicc_for_sleep
     /* Set OS double lock */
@@ -267,11 +278,11 @@ finalize_powerdown:
 wait_for_power_off:
     wfi
     b wait_for_power_off
-    
+
 /* Call a function with desired stack pointer. */
 .section    .text.call_with_stack_pointer, "ax", %progbits
 .type       call_with_stack_pointer, %function
 .global     call_with_stack_pointer
 call_with_stack_pointer:
     mov sp, x0
-    br x1
+    br x1

exosphere/src/coldboot_init.c

@@ -12,7 +12,7 @@ extern const uint8_t __vectors_start__[], __vectors_end__[], __vectors_lma__[];
 /* warmboot_init.c */
 void set_memory_registers_enable_mmu(void);
 void flush_dcache_all_tzram_pa(void);
-void invalidate_icache_all_tzram_pa(void);
+void invalidate_icache_all_inner_shareable_tzram_pa(void);
 
 static void identity_map_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t *mmu_l3_tbl) {
     static const uintptr_t addrs[]      =   { TUPLE_FOLD_LEFT_0(EVAL(IDENTIY_MAPPING_ID_MAX), _MMAPID, COMMA) };
@@ -130,7 +130,7 @@ void coldboot_init(void) {
     set_memory_registers_enable_mmu();
 
     flush_dcache_all_tzram_pa();
-    invalidate_icache_all_tzram_pa();
+    invalidate_icache_all_inner_shareable_tzram_pa();
     /* At this point we can access all the mapped segments */
     /* TODO: zero-initialize the cpu context */
     /* Nintendo clears the (emtpy) pk2ldr's BSS section here , but we embed it 0-filled in the binary */

exosphere/src/package2.c

@@ -19,7 +19,7 @@ static void setup_se(void) {
 
     /* Sanity check the Security Engine. */
     se_verify_flags_cleared();
-    
+
     /* Initialize Interrupts. */
     intr_initialize_gic_nonsecure();
 
@@ -380,7 +380,7 @@ void load_package2(void) {
 
     /* Clean up cache. */
     flush_dcache_all();
-    invalidate_icache_inner_shareable();
+    invalidate_icache_all_inner_shareable();
 
     /* Set CORE0 entrypoint for Package2. */
     set_core_entrypoint_and_argument(0, DRAM_BASE_PHYSICAL + header.metadata.entrypoint, 0);

exosphere/src/warmboot_init.c

@@ -1,16 +1,25 @@
 #include "utils.h"
 #include "memory_map.h"
+#include "arm.h"
+
+extern const uint8_t __main_start__[];
 
 /* start.s */
 void __set_memory_registers(uintptr_t ttbr0, uintptr_t vbar, uint64_t cpuectlr, uint32_t scr,
                             uint32_t tcr, uint32_t cptr, uint64_t mair, uint32_t sctlr);
 
-void flush_dcache_all_tzram_pa(void) {
+__attribute__((target("cmodel=large"))) void flush_dcache_all_tzram_pa(void) {
-    /* TODO */
+    uintptr_t pa = TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_WARMBOOT_CRT0_AND_MAIN);
+    uintptr_t main_pa = pa | ((uintptr_t)__main_start__ & 0xFFF);
+    uintptr_t v = (uintptr_t)flush_dcache_all - (uintptr_t)__main_start__ + (uintptr_t)main_pa;
+    ((void (*)(void))v)();
 }
 
-void invalidate_icache_all_tzram_pa(void) {
+__attribute__((target("cmodel=large"))) void invalidate_icache_all_inner_shareable_tzram_pa(void) {
-    /* TODO */
+    uintptr_t pa = TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_WARMBOOT_CRT0_AND_MAIN);
+    uintptr_t main_pa = pa | ((uintptr_t)__main_start__ & 0xFFF);
+    uintptr_t v = (uintptr_t)invalidate_icache_all_inner_shareable - (uintptr_t)__main_start__ + (uintptr_t)main_pa;
+    ((void (*)(void))v)();
 }
 
 uintptr_t get_warmboot_crt0_stack_address(void) {