diff --git a/thermosphere/Makefile b/thermosphere/Makefile
index 4d1c07495..6b5836500 100644
--- a/thermosphere/Makefile
+++ b/thermosphere/Makefile
@@ -80,7 +80,7 @@ CXXFLAGS := $(CFLAGS) -fno-rtti -fno-exceptions -std=gnu++11
ASFLAGS := -g $(ARCH)
LDFLAGS = -specs=$(TOPDIR)/linker.specs -nostartfiles -nostdlib -g $(ARCH) -Wl,-Map,$(notdir $*.map)
-LIBS :=
+LIBS := -lgcc
#---------------------------------------------------------------------------------
# list of directories containing libraries, this must be the top level containing
diff --git a/thermosphere/src/breakpoints.c b/thermosphere/src/breakpoints.c
index 94e03a199..b7b7a2cf1 100644
--- a/thermosphere/src/breakpoints.c
+++ b/thermosphere/src/breakpoints.c
@@ -58,7 +58,7 @@ static void freeBreakpoint(u32 pos)
static DebugRegisterPair *findBreakpoint(u64 addr)
{
- u16 bitmap = g_breakpointManager.allocationBitmap;
+ u16 bitmap = ~g_breakpointManager.allocationBitmap & 0xFFFF;
while (bitmap != 0) {
u32 pos = __builtin_ffs(bitmap);
if (pos == 0) {
@@ -119,6 +119,7 @@ bool removeBreakpoint(u64 addr)
DebugRegisterPair *regs = findBreakpoint(addr);
if (findBreakpoint(addr) == NULL) {
+ recursiveSpinlockUnlock(&g_breakpointManager.lock);
return false;
}
@@ -128,4 +129,4 @@ bool removeBreakpoint(u64 addr)
// TODO commit & broadcast
return true;
-}
\ No newline at end of file
+}
diff --git a/thermosphere/src/main.c b/thermosphere/src/main.c
index 59687ad31..342f78a90 100644
--- a/thermosphere/src/main.c
+++ b/thermosphere/src/main.c
@@ -42,12 +42,15 @@ void main(ExceptionStackFrame *frame)
enableTraps();
enableBreakpointsAndWatchpoints();
+ if (currentCoreCtx->isBootCore) {
+ uartInit(115200);
+ DEBUG("EL2: core %u reached main first!\n", currentCoreCtx->coreId);
+ }
+
initBreakpoints();
initWatchpoints();
if (currentCoreCtx->isBootCore) {
- uartInit(115200);
- DEBUG("EL2: core %u reached main first!\n", currentCoreCtx->coreId);
if (currentCoreCtx->kernelEntrypoint == 0) {
if (semihosting_connection_supported()) {
loadKernelViaSemihosting();
diff --git a/thermosphere/src/watchpoints.c b/thermosphere/src/watchpoints.c
index d783b4971..8e973d696 100644
--- a/thermosphere/src/watchpoints.c
+++ b/thermosphere/src/watchpoints.c
@@ -14,13 +14,18 @@
* along with this program. If not, see .
*/
+#include
#include "watchpoints.h"
#include "breakpoints_watchpoints_save_restore.h"
#include "utils.h"
#include "sysreg.h"
#include "arm.h"
+#include "debug_log.h"
WatchpointManager g_watchpointManager = {0};
+static TEMPORARY DebugRegisterPair g_combinedWatchpoints[16] = {0};
+
+static void combineAllCurrentWatchpoints(void);
// Init the structure (already in BSS, so already zero-initialized) and load the registers
void initWatchpoints(void)
@@ -30,10 +35,266 @@ void initWatchpoints(void)
if (currentCoreCtx->isBootCore && !currentCoreCtx->warmboot) {
size_t num = ((GET_SYSREG(id_aa64dfr0_el1) >> 20) & 0xF) + 1;
g_watchpointManager.maxWatchpoints = (u32)num;
- g_watchpointManager.allocationBitmap = 0xFFFF;
+ g_watchpointManager.maxSplitWatchpoints = 8 * (u32)num;
+ g_watchpointManager.allocationBitmap = BIT(num) - 1;
+ } else if (currentCoreCtx->isBootCore) {
+ combineAllCurrentWatchpoints();
}
- loadWatchpointRegs(g_watchpointManager.watchpoints, g_watchpointManager.maxWatchpoints);
+ loadWatchpointRegs(g_combinedWatchpoints, g_watchpointManager.maxWatchpoints);
recursiveSpinlockUnlock(&g_watchpointManager.lock);
}
+
+static DebugRegisterPair *findCombinedWatchpoint(u64 addr)
+{
+ addr &= ~7ull;
+ u16 bitmap = ~g_watchpointManager.allocationBitmap & 0xFFFF;
+ while (bitmap != 0) {
+ u32 pos = __builtin_ffs(bitmap);
+ if (pos == 0) {
+ return NULL;
+ } else {
+ bitmap &= ~BIT(pos - 1);
+ if (g_combinedWatchpoints[pos - 1].vr == addr) {
+ return &g_combinedWatchpoints[pos - 1];
+ }
+ }
+ }
+
+ return NULL;
+}
+
+static DebugRegisterPair *allocateCombinedWatchpoint(u16 *bitmap)
+{
+ u32 pos = __builtin_ffs(*bitmap);
+ if (pos == 0) {
+ return NULL;
+ } else {
+ *bitmap &= ~BIT(pos - 1);
+ return &g_combinedWatchpoints[pos - 1];
+ }
+}
+
+// Precondition: not a MASK-based watchpoint
+static bool checkNormalWatchpointRange(u64 addr, size_t size)
+{
+ u16 bitmap = g_watchpointManager.allocationBitmap;
+ if (findCombinedWatchpoint(addr) == NULL) {
+ if (allocateCombinedWatchpoint(&bitmap) == NULL) {
+ return false;
+ }
+ }
+
+ // if it overlaps...
+ u64 addr2 = (addr + size) & ~7ull;
+
+ if (addr2 != (addr & ~7ull)) {
+ if (findCombinedWatchpoint(addr2) == NULL) {
+ return allocateCombinedWatchpoint(&bitmap) != NULL;
+ }
+ }
+
+ return true;
+}
+
+static inline bool isRangeMaskWatchpoint(u64 addr, size_t size)
+{
+ // size needs to be a power of 2, at least 8 (we'll only allow 16+ though), addr needs to be aligned.
+ bool ret = (size & (size - 1)) == 0 && size >= 16 && (addr & (size - 1)) == 0;
+ return ret;
+}
+
+static bool combineWatchpoint(const DebugRegisterPair *wp)
+{
+ DebugRegisterPair *wpSlot = NULL;
+
+ wpSlot = findCombinedWatchpoint(wp->vr & ~7ull);
+
+ // To simplify, don't allow combining for wps that use MASK (except if only perms needs to be combined)
+ if (wp->cr.mask != 0 && wpSlot != NULL && (wp->cr.mask != wpSlot->cr.mask || wp->vr != wpSlot->vr)) {
+ wpSlot = NULL;
+ }
+
+ if (wpSlot == NULL) {
+ wpSlot = allocateCombinedWatchpoint(&g_watchpointManager.allocationBitmap);
+ memset(wpSlot, 0, sizeof(DebugRegisterPair));
+ wpSlot->vr = wp->vr & ~7ull;
+ }
+
+ if (wpSlot == NULL) {
+ return false;
+ }
+
+ wpSlot->cr.hmc = DebugHmc_LowerEl;
+ wpSlot->cr.ssc = DebugSsc_NonSecure;
+ wpSlot->cr.pmc = DebugPmc_El1And0;
+ wpSlot->cr.enabled = true;
+
+ // Merge 8-byte selection mask and access permissions (possibly broadening them)
+ wpSlot->cr.bas |= wp->cr.bas;
+ wpSlot->cr.lsc |= wp->cr.lsc;
+
+ return true;
+}
+
+static DebugRegisterPair *doFindSplitWatchpoint(u64 addr, size_t size, WatchpointLoadStoreControl direction, bool strict)
+{
+ // Note: we will use RES0 bit0_1 of wr in case of overlapping
+ for (u32 i = 0; i < g_watchpointManager.numSplitWatchpoints; i++) {
+ DebugRegisterPair *wp = &g_watchpointManager.splitWatchpoints[i];
+ if (wp->vr & 2) {
+ continue;
+ }
+
+ size_t off = 0;
+ size_t sz = 0;
+
+ if (wp->cr.mask != 0) {
+ off = 0;
+ sz = size;
+ } else {
+ off = __builtin_ffs(wp->cr.bas) - 1;
+ sz = __builtin_popcount(wp->cr.bas);
+ if (wp->vr & 1) {
+ DebugRegisterPair *wp2 = &g_watchpointManager.splitWatchpoints[i + 1];
+ sz += __builtin_popcount(wp2->cr.bas);
+ }
+ }
+
+ u64 wpaddr = (wp->vr & ~7ull) + off;
+ if (strict) {
+ if (addr == wpaddr && direction == wp->cr.lsc && sz == size) {
+ return wp;
+ }
+ } else if (overlaps(wpaddr, sz, addr, size) && (direction & wp->cr.lsc) != 0) {
+ return wp;
+ }
+ }
+
+ return NULL;
+}
+
+DebugRegisterPair *findSplitWatchpoint(u64 addr, size_t size, WatchpointLoadStoreControl direction, bool strict)
+{
+ recursiveSpinlockLock(&g_watchpointManager.lock);
+ DebugRegisterPair *ret = doFindSplitWatchpoint(addr, size, direction, strict);
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return ret;
+}
+
+bool addWatchpoint(u64 addr, size_t size, WatchpointLoadStoreControl direction)
+{
+ if (size == 0) {
+ return false;
+ }
+
+ recursiveSpinlockLock(&g_watchpointManager.lock);
+
+ if (doFindSplitWatchpoint(addr, size, direction, true)) {
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return true;
+ }
+
+ if (g_watchpointManager.numSplitWatchpoints == g_watchpointManager.maxSplitWatchpoints) {
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return false;
+ }
+
+ size_t oldNumSplitWatchpoints = g_watchpointManager.numSplitWatchpoints;
+ DebugRegisterPair *wp = &g_watchpointManager.splitWatchpoints[g_watchpointManager.numSplitWatchpoints++], *wp2 = NULL;
+
+ memset(wp, 0, sizeof(DebugRegisterPair));
+ wp->cr.lsc = direction;
+ if (isRangeMaskWatchpoint(addr, size)) {
+ wp->vr = addr;
+ wp->cr.bas = 0xFF; // TRM-mandated
+ wp->cr.mask = (u32)__builtin_ffsl(size) - 1;
+ if (!combineWatchpoint(wp)) {
+ g_watchpointManager.numSplitWatchpoints = oldNumSplitWatchpoints;
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return false;
+ }
+ } else if (size <= 9) {
+ // Normal one or 2 up-to-9-bytes wp(s) (ie. never exceeeds two combined wp)
+ // Note: we will use RES0 bit0_1 of wr in case of overlapping
+ if (!checkNormalWatchpointRange(addr, size)) {
+ g_watchpointManager.numSplitWatchpoints = oldNumSplitWatchpoints;
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return false;
+ }
+
+ u64 addr2 = (addr + size) & ~7ull;
+ size_t off1 = addr & 7ull;
+ size_t size1 = (addr != addr2) ? 8 - off1 : size;
+ size_t size2 = size - size1;
+ wp->vr = addr & ~7ull;
+ wp->cr.bas = MASK2(off1 + size1, off1);
+
+ if (size2 != 0) {
+ if (g_watchpointManager.numSplitWatchpoints == g_watchpointManager.maxSplitWatchpoints) {
+ g_watchpointManager.numSplitWatchpoints = oldNumSplitWatchpoints;
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return false;
+ }
+ wp2 = &g_watchpointManager.splitWatchpoints[g_watchpointManager.numSplitWatchpoints++];
+ wp2->cr.lsc = direction;
+ wp2->cr.bas = MASK2(size2, 0);
+
+ // Note: we will use RES0 bit0_1 of wr in case of overlapping
+ wp->vr |= 1;
+ wp2->vr = addr2 | 2;
+ }
+
+ if (!combineWatchpoint(wp) || (size2 != 0 && !combineWatchpoint(wp2))) {
+ g_watchpointManager.numSplitWatchpoints = oldNumSplitWatchpoints;
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return false;
+ }
+ } else {
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+ return false;
+ }
+
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+
+ // TODO: commit and broadcast
+
+ return true;
+}
+
+static void combineAllCurrentWatchpoints(void)
+{
+ memset(g_combinedWatchpoints, 0, sizeof(g_combinedWatchpoints));
+ g_watchpointManager.allocationBitmap = BIT(g_watchpointManager.maxWatchpoints) - 1;
+ for (u32 i = 0; i < g_watchpointManager.numSplitWatchpoints; i++) {
+ combineWatchpoint(&g_watchpointManager.splitWatchpoints[i]);
+ }
+}
+
+bool removeWatchpoint(u64 addr, size_t size, WatchpointLoadStoreControl direction)
+{
+ if (size == 0) {
+ return false;
+ }
+
+ recursiveSpinlockLock(&g_watchpointManager.lock);
+
+ DebugRegisterPair *wp = doFindSplitWatchpoint(addr, size, direction, true);
+ if (wp != NULL) {
+ size_t pos = wp - &g_watchpointManager.splitWatchpoints[0];
+ size_t num = (wp->vr & 1) ? 2 : 1;
+ for (size_t i = pos + num; i < g_watchpointManager.numSplitWatchpoints; i ++) {
+ g_watchpointManager.splitWatchpoints[i - num] = g_watchpointManager.splitWatchpoints[i];
+ }
+ g_watchpointManager.numSplitWatchpoints -= num;
+ combineAllCurrentWatchpoints();
+ } else {
+ DEBUG("watchpoint not found 0x%016llx, size %llu, direction %d\n", addr, size, direction);
+ }
+ recursiveSpinlockUnlock(&g_watchpointManager.lock);
+
+ // TODO: commit and broadcast
+
+ return true;
+}
diff --git a/thermosphere/src/watchpoints.h b/thermosphere/src/watchpoints.h
index 12a6a5e89..899510a50 100644
--- a/thermosphere/src/watchpoints.h
+++ b/thermosphere/src/watchpoints.h
@@ -21,12 +21,17 @@
/// Structure to synchronize and keep track of watchpoints
typedef struct WatchpointManager {
- DebugRegisterPair watchpoints[16];
+ DebugRegisterPair splitWatchpoints[16 * 8];
RecursiveSpinlock lock;
+ u32 numSplitWatchpoints;
u32 maxWatchpoints;
+ u32 maxSplitWatchpoints;
u16 allocationBitmap;
} WatchpointManager;
extern WatchpointManager g_watchpointManager;
void initWatchpoints(void);
+DebugRegisterPair *findSplitWatchpoint(u64 addr, size_t size, WatchpointLoadStoreControl direction, bool strict);
+bool addWatchpoint(u64 addr, size_t size, WatchpointLoadStoreControl direction);
+bool removeWatchpoint(u64 addr, size_t size, WatchpointLoadStoreControl direction);