diff --git a/thermosphere/src/defines.hpp b/thermosphere/src/defines.hpp index 4a19bf083..f712c521a 100644 --- a/thermosphere/src/defines.hpp +++ b/thermosphere/src/defines.hpp @@ -34,3 +34,8 @@ static cl instance;\ public:\ static cl &GetInstance() { return instance; } + +//FIXME +#ifndef ENSURE +#define ENSURE(...) +#endif diff --git a/thermosphere/src/hvisor_irq_manager.cpp b/thermosphere/src/hvisor_irq_manager.cpp index d0b30f8b9..8b0f36b6a 100644 --- a/thermosphere/src/hvisor_irq_manager.cpp +++ b/thermosphere/src/hvisor_irq_manager.cpp @@ -151,9 +151,9 @@ namespace ams::hvisor { ClearInterruptPending(id); if (id >= 32) { SetInterruptMode(id, isLevelSensitive); - DoSetInterruptAffinity(id, 0xFF); // all possible processors + SetInterruptTargets(id, 0xFF); // all possible processors } - SetInterruptShiftedPriority(id, prio << m_priorityShift); + SetInterruptPriority(id, prio); SetInterruptEnabled(id); } @@ -185,7 +185,7 @@ namespace ams::hvisor { cpu::InterruptMaskGuard mg{}; std::scoped_lock lk{m_lock}; - DoSetInterruptAffinity(id, affinity); + SetInterruptTargets(id, affinity); } void IrqManager::HandleInterrupt(ExceptionStackFrame *frame) diff --git a/thermosphere/src/hvisor_irq_manager.hpp b/thermosphere/src/hvisor_irq_manager.hpp index 78364422e..3784ee7bc 100644 --- a/thermosphere/src/hvisor_irq_manager.hpp +++ b/thermosphere/src/hvisor_irq_manager.hpp @@ -37,11 +37,12 @@ namespace ams::hvisor { static bool IsGuestInterrupt(u32 id); + static u32 GetTypeRegister() { return gicd->typer; } static void SetInterruptEnabled(u32 id) { gicd->isenabler[id / 32] = BIT(id % 32); } static void ClearInterruptEnabled(u32 id) { gicd->icenabler[id / 32] = BIT(id % 32); } static void ClearInterruptPending(u32 id) { gicd->icpendr[id / 32] = BIT(id % 32); } static void SetInterruptShiftedPriority(u32 id, u8 prio) { gicd->ipriorityr[id] = prio; } - static void DoSetInterruptAffinity(u32 id, u8 targetList) { gicd->itargetsr[id] = targetList; } + static void SetInterruptTargets(u32 id, u8 targetList) { gicd->itargetsr[id] = targetList; } static bool IsInterruptLevelSensitive(u32 id) { return ((gicd->icfgr[id / 16] >> GicV2Distributor::GetCfgrShift(id)) & 2) != 0; @@ -50,7 +51,7 @@ namespace ams::hvisor { { u32 cfgw = gicd->icfgr[id / 16]; cfgw &= ~(2 << GicV2Distributor::GetCfgrShift(id)); - cfgw |= (isLevelSensitive ? 2 : 0) << GicV2Distributor::GetCfgrShift(id); + cfgw |= (!isLevelSensitive ? 2 : 0) << GicV2Distributor::GetCfgrShift(id); gicd->icfgr[id / 16] = cfgw; } @@ -67,6 +68,8 @@ namespace ams::hvisor { u8 m_numListRegisters = 0; private: + void SetInterruptPriority(u32 id, u8 prio) { SetInterruptShiftedPriority(id, prio << m_priorityShift); } + void InitializeGic(); void DoConfigureInterrupt(u32 id, u8 prio, bool isLevelSensitive); diff --git a/thermosphere/src/hvisor_virtual_gic.cpp b/thermosphere/src/hvisor_virtual_gic.cpp new file mode 100644 index 000000000..7cccc6554 --- /dev/null +++ b/thermosphere/src/hvisor_virtual_gic.cpp @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "hvisor_virtual_gic.hpp" + +namespace ams::hvisor { + + + VirtualGic::VirqQueue::iterator VirtualGic::VirqQueue::insert(VirtualGic::VirqQueue::iterator pos, VirtualGic::VirqState &elem) + { + // Insert before + ENSURE(!elem.IsQueued()); + + // Empty list + if (begin() == end()) { + m_first = m_last = &elem; + elem.listPrev = elem.listNext = virqListEndIndex; + return begin(); + } + + if (pos == end()) { + // Insert after last + VirqState &prev = back(); + elem.listPrev = GetStateIndex(prev); + elem.listNext = prev.listNext; + prev.listNext = GetStateIndex(elem); + m_last = &elem; + } else { + u32 idx = GetStateIndex(elem); + u32 posidx = GetStateIndex(*pos); + u32 previdx = elem.listPrev; + + elem.listNext = posidx; + elem.listPrev = previdx; + + pos->listPrev = idx; + + if (pos == begin()) { + m_first = &elem; + } else { + --pos; + pos->listNext = idx; + } + } + + return iterator{&elem, m_storage}; + } + + VirtualGic::VirqQueue::iterator VirtualGic::VirqQueue::insert(VirtualGic::VirqState &elem) + { + // Insert in a stable sorted way + // Lower priority number is higher; we sort by descending priority, ie. ascending priority number + // Put the interrupts that were previously in the LR before the one which don't + return insert( + std::find_if(begin(), end(), [&a = elem](const VirqState &b) { + return a.priority == b.priority ? a.handled && !b.handled : a.priority < b.priority; + }), + elem + ); + } + + VirtualGic::VirqQueue::iterator VirtualGic::VirqQueue::erase(VirtualGic::VirqQueue::iterator startPos, VirtualGic::VirqQueue::iterator endPos) + { + VirqState &prev = m_storage[startPos->listPrev]; + VirqState &next = *endPos; + u32 nextPos = GetStateIndex(*endPos); + + if (startPos->listPrev != virqListEndIndex) { + prev.listNext = nextPos; + } else { + m_first = &next; + } + + if (nextPos != virqListEndIndex) { + next.listPrev = startPos->listPrev; + } else { + m_last = &prev; + } + + for (auto it = startPos; it != endPos; ++it) { + it->listPrev = it->listNext = virqListInvalidIndex; + } + } + + void VirtualGic::SetInterruptEnabledState(u32 id) + { + VirqState &state = GetVirqState(id); + + if (id < 16 || !IrqManager::IsGuestInterrupt(id) || state.enabled) { + // Nothing to do... + // Also, ignore for SGIs + return; + } + + // Similar effects to setting the target list to non-0 when it was 0... + if (state.IsPending()) { + NotifyOtherCoreList(state.targetList); + } + + state.enabled = true; + IrqManager::SetInterruptEnabled(id); + } + + void VirtualGic::ClearInterruptEnabledState(u32 id) + { + VirqState &state = GetVirqState(id); + + if (id < 16 || !IrqManager::IsGuestInterrupt(id) || !state.enabled) { + // Nothing to do... + // Also, ignore for SGIs + return; + } + + // Similar effects to setting the target list to 0, we may need to notify the core + // handling the interrupt if it's pending + if (state.handled) { + NotifyOtherCoreList(BIT(state.coreId)); + } + + state.enabled = false; + IrqManager::ClearInterruptEnabled(id); + } + + void VirtualGic::SetInterruptPriorityByte(u32 id, u8 priority) + { + if (!IrqManager::IsGuestInterrupt(id)) { + return; + } + + // 32 priority levels max, bits [7:3] + priority >>= priorityShift; + + if (id >= 16) { + // Ensure we have the correct priority on the physical distributor... + IrqManager::GetInstance().SetInterruptPriority(id, IrqManager::guestPriority); + } + + VirqState &state = GetVirqState(id); + if (priority == state.priority) { + // Nothing to do... + return; + } + + state.priority = priority; + u32 targets = state.targetList; + if (targets != 0 && state.IsPending()) { + NotifyOtherCoreList(targets); + } + } + + void VirtualGic::SetInterruptTargets(u32 id, u8 coreList) + { + // Ignored for SGIs and PPIs, and non-guest interrupts + if (id < 32 || !IrqManager::IsGuestInterrupt(id)) { + return; + } + + // Interrupt not pending (inactive or active-only): nothing much to do (see reference manual) + // Otherwise, we may need to migrate the interrupt. + // In our model, while a physical interrupt can be pending on multiple cores, we decide that a pending SPI + // can only be handled on a single core (either it's in a LR, or in the global list), therefore we need to + // send a signal to (oldList XOR newList) to either put the interrupt back in the global list or potentially handle it + + // Note that we take into account that the interrupt may be disabled. + VirqState &state = GetVirqState(id); + if (state.IsPending()) { + u8 oldList = state.targetList; + u8 diffList = (oldList ^ coreList) & getActiveCoreMask(); + if (diffList != 0) { + NotifyOtherCoreList(diffList); + } + } + + state.targetList = coreList; + IrqManager::SetInterruptTargets(id, state.targetList); + } + + void VirtualGic::SetInterruptConfigBits(u32 id, u32 config) + { + // Ignored for SGIs, implementation defined for PPIs + if (id < 32 || !IrqManager::IsGuestInterrupt(id)) { + return; + } + + VirqState &state = GetVirqState(id); + + // Expose bit(2n) as nonprogrammable to the guest no matter what the physical distributor actually behaves + bool newLvl = ((config & 2) << GicV2Distributor::GetCfgrShift(id)) == 0; + + if (state.levelSensitive != newLvl) { + state.levelSensitive = newLvl; + IrqManager::SetInterruptMode(id, newLvl); + } + } + + void VirtualGic::SetSgiPendingState(u32 id, u32 coreId, u32 srcCoreId) + { + VirqState &state = GetVirqState(coreId, id); + m_incomingSgiPendingSources[coreId][id] |= BIT(srcCoreId); + if (!state.handled && !state.IsQueued()) { + // The SGI was inactive on the target core... + state.SetPending(); + state.srcCoreId = srcCoreId; + m_incomingSgiPendingSources[coreId][id] &= ~BIT(srcCoreId); + m_virqPendingQueue.insert(state); + NotifyOtherCoreList(BIT(coreId)); + } + } + + void VirtualGic::SendSgi(u32 id, GicV2Distributor::SgirTargetListFilter filter, u32 coreList) + { + switch (filter) { + case GicV2Distributor::ForwardToTargetList: + // Forward to coreList + break; + case GicV2Distributor::ForwardToAllOthers: + // Forward to all but current core + coreList = ~BIT(currentCoreCtx->coreId); + break; + case GicV2Distributor::ForwardToSelf: + // Forward to current core only + coreList = BIT(currentCoreCtx->coreId); + break; + default: + DEBUG("Emulated GCID_SGIR: invalid TargetListFilter value!\n"); + return; + } + + coreList &= getActiveCoreMask(); + for (u32 dstCore: util::BitsOf{coreList}) { + SetSgiPendingState(id, dstCore, currentCoreCtx->coreId); + } + + } +} diff --git a/thermosphere/src/hvisor_virtual_gic.hpp b/thermosphere/src/hvisor_virtual_gic.hpp new file mode 100644 index 000000000..a1cd5732d --- /dev/null +++ b/thermosphere/src/hvisor_virtual_gic.hpp @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2019-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "defines.hpp" +#include "exceptions.h" +#include "cpu/hvisor_cpu_exception_sysregs.hpp" +#include "hvisor_irq_manager.hpp" +#include "memory_map.h" + +namespace ams::hvisor { + + class VirtualGic final { + SINGLETON(VirtualGic); + + private: + // For convenience, although they're already defined in irq manager header: + static inline volatile auto *const gicd = IrqManager::gicd; + static inline volatile auto *const gich = IrqManager::gich; + + // Architectural properties + static constexpr u32 priorityShift = 3; + + // List managament constants + static constexpr u32 spiEndIndex = GicV2Distributor::maxIrqId + 1 - 32; + static constexpr u32 maxNumIntStates = spiEndIndex + MAX_CORE * 32; + static constexpr u32 virqListEndIndex = maxNumIntStates; + static constexpr u32 virqListInvalidIndex = virqListEndIndex + 1; + + private: + struct VirqState { + u32 listPrev : 11; + u32 listNext : 11; + u32 irqId : 10; + u32 priority : 5; + bool pending : 1; + bool active : 1; + bool handled : 1; + bool pendingLatch : 1; + bool levelSensitive : 1; + u32 coreId : 3; + u32 targetList : 8; + u32 srcCoreId : 3; + bool enabled : 1; + u64 : 0; + + constexpr bool IsPending() const + { + return pendingLatch || (levelSensitive && pending); + } + constexpr void SetPending() + { + if (levelSensitive) { + pending = true; + } else { + pendingLatch = true; + } + } + constexpr bool ClearPendingLine() + { + // Don't clear pending latch status + pending = false; + } + constexpr bool ClearPending() + { + // On ack, both pending line status and latch are cleared + pending = false; + pendingLatch = false; + } + constexpr bool IsQueued() const + { + return listPrev != virqListInvalidIndex && listNext != virqListInvalidIndex; + } + }; + + + class VirqQueue final { + private: + VirqState *m_first = nullptr; + VirqState *m_last = nullptr; + VirqState *m_storage = nullptr; + public: + template + class Iterator { + friend class Iterator; + friend class VirqQueue; + private: + VirqState *m_node = nullptr; + VirqState *m_storage = nullptr; + + private: + explicit constexpr Iterator(VirqState *node, VirqState *storage) : m_node{node}, m_storage{storage} {} + + public: + // allow implicit const->non-const + constexpr Iterator(const Iterator &other) : m_node{other.m_storage}, m_storage{other.m_storage} {} + constexpr Iterator() = default; + + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = VirqState; + using difference_type = ptrdiff_t; + using pointer = typename std::conditional::type; + using reference = typename std::conditional::type; + + constexpr bool operator==(const Iterator &other) const { return m_node == other.m_node; } + constexpr bool operator!=(const Iterator &other) const { return !(*this == other); } + constexpr reference operator*() { return *m_node; } + constexpr pointer operator->() { return m_node; } + + constexpr Iterator &operator++() + { + m_node = &m_storage[m_node->listNext]; + return *this; + } + + constexpr Iterator &operator--() + { + m_node = &m_storage[m_node->listPrev]; + return *this; + } + + constexpr Iterator &operator++(int) + { + const Iterator v{*this}; + ++(*this); + return v; + } + + constexpr Iterator &operator--(int) + { + const Iterator v{*this}; + --(*this); + return v; + } + }; + + private: + constexpr u32 GetStateIndex(VirqState &elem) { return static_cast(&elem - &m_storage[0]); } + public: + using pointer = VirqState *; + using const_pointer = const VirqState *; + using reference = VirqState &; + using const_reference = const VirqState &; + using value_type = VirqState; + using size_type = size_t; + using difference_type = ptrdiff_t; + using iterator = Iterator; + using const_iterator = Iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + constexpr void Initialize(VirqState *storage) { m_storage = storage; } + + constexpr VirqState &front() { return *m_first; }; + constexpr const VirqState &front() const { return *m_first; }; + + constexpr VirqState &back() { return *m_last; }; + constexpr const VirqState &back() const { return *m_last; }; + + constexpr const_iterator cbegin() const { return const_iterator{m_first, m_storage}; } + constexpr const_iterator cend() const { return const_iterator{&m_storage[virqListEndIndex], m_storage}; } + + constexpr const_iterator begin() const { return cbegin(); } + constexpr const_iterator end() const { return cend(); } + + constexpr iterator begin() { return iterator{m_first, m_storage}; } + constexpr iterator end() { return iterator{&m_storage[virqListEndIndex], m_storage}; } + + constexpr const_reverse_iterator crbegin() const { + return const_reverse_iterator{const_iterator{m_last, m_storage}}; + } + constexpr const_reverse_iterator crend() const { return const_reverse_iterator{cend()}; } + + constexpr const_reverse_iterator rbegin() const { return crbegin(); } + constexpr const_reverse_iterator rend() const { return crend(); } + + constexpr reverse_iterator rbegin() { return reverse_iterator{iterator{m_first, m_storage}}; } + constexpr reverse_iterator rend() { return reverse_iterator{end()}; } + + + iterator insert(iterator pos, VirqState &elem); + iterator insert(VirqState &elem); + + iterator erase(iterator startPos, iterator endPos); + + iterator erase(iterator pos) { return erase(pos, std::next(pos)); } + }; + + + private: + static void NotifyOtherCoreList(u32 coreList) + { + coreList &= ~BIT(currentCoreCtx->coreId); + if (coreList != 0) { + IrqManager::GenerateSgiForList(IrqManager::VgicUpdateSgi, coreList); + } + } + + static void NotifyAllOtherCores() + { + IrqManager::GenerateSgiForAllOthers(IrqManager::VgicUpdateSgi); + } + + + private: + std::array m_virqStates{}; + std::array, MAX_CORE> m_incomingSgiPendingSources{}; + + VirqQueue m_virqPendingQueue{}; + bool m_distributorEnabled = false; + + private: + + constexpr VirqState &GetVirqState(u32 coreId, u32 id) + { + if (id >= 32) { + return m_virqStates[id - 32]; + } else if (id <= GicV2Distributor::maxIrqId) { + return m_virqStates[spiEndIndex + 32 * coreId + id]; + } + } + + VirqState &GetVirqState(u32 id) { return GetVirqState(currentCoreCtx->coreId, id); } + + void SetDistributorControlRegister(u32 value) + { + // We implement a virtual distributor/interface w/o security extensions. + // Moreover, we forward all interrupts as Group 0 so that non-secure code that assumes GICv2 + // *with* security extensions (and thus all interrupts fw as group 1 there) still works (bit are in the same positions). + + // We don't implement Group 1 interrupts, either (so that's similar to GICv1). + bool old = m_distributorEnabled; + m_distributorEnabled = (value & 1) != 0; + + // Enable bit is actually just a global enable bit for all irq forwarding, other functions of the GICD aren't affected by it + if (old != m_distributorEnabled) { + NotifyAllOtherCores(); + } + } + + u32 vgicGetDistributorControlRegister(void) + { + return m_distributorEnabled ? 1 : 0; + } + + u32 vgicGetDistributorTypeRegister(void) + { + // See above comment. + // Therefore, LSPI = 0, SecurityExtn = 0, rest = from physical distributor + return IrqManager::GetTypeRegister() & 0x7F; + } + + u32 GetDistributorImplementerIdentificationRegister(void) + { + u32 iidr = 'A' << 24; // Product Id: Atmosphère (?) + iidr |= 2 << 16; // Major revision 2 (GICv2) + iidr |= 0 << 12; // Minor revision 0 + iidr |= 0x43B; // Implementer: Arm (value copied from physical GICD) + return iidr; + } + + bool GetInterruptEnabledState(u32 id) + { + // SGIs are always enabled + return id < 16 || (IrqManager::IsGuestInterrupt(id) && GetVirqState(currentCoreCtx->coreId, id).enabled); + } + + u8 GetInterruptPriorityByte(u32 id) + { + return IrqManager::IsGuestInterrupt(id) ? GetVirqState(currentCoreCtx->coreId, id).priority << priorityShift : 0; + } + + u8 GetInterruptTargets(u16 id) + { + return id < 32 || (IrqManager::IsGuestInterrupt(id) && GetVirqState(currentCoreCtx->coreId, id).targetList); + } + + u32 GetInterruptConfigBits(u16 id) + { + u32 oneNModel = id < 32 || !IrqManager::IsGuestInterrupt(id) ? 0 : 1; + return (IrqManager::IsGuestInterrupt(id) && !GetVirqState(id).levelSensitive) ? 2 | oneNModel : oneNModel; + } + + u32 GetPeripheralId2Register(void) + { + return 2u << 4; + } + + void SetInterruptEnabledState(u32 id); + void ClearInterruptEnabledState(u32 id); + void SetInterruptPriorityByte(u32 id, u8 priority); + void SetInterruptTargets(u32 id, u8 coreList); + void SetInterruptConfigBits(u32 id, u32 config); + void SetSgiPendingState(u32 id, u32 coreId, u32 srcCoreId); + void SendSgi(u32 id, GicV2Distributor::SgirTargetListFilter filter, u32 coreList); + + }; +} + + +/*bool vgicValidateGicdRegisterAccess(size_t offset, size_t sz); +void vgicWriteGicdRegister(u32 val, size_t offset, size_t sz); +u32 vgicReadGicdRegister(size_t offset, size_t sz); + +void handleVgicdMmio(ExceptionStackFrame *frame, cpu::DataAbortIss dabtIss, size_t offset); + +void vgicInit(void); +void vgicUpdateState(void); +void vgicMaintenanceInterruptHandler(void); +void vgicEnqueuePhysicalIrq(u16 irqId);*/ diff --git a/thermosphere/src/vgic.h b/thermosphere/src/vgic.h deleted file mode 100644 index 57402e25c..000000000 --- a/thermosphere/src/vgic.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "types.h" -#include "data_abort.h" - -bool vgicValidateGicdRegisterAccess(size_t offset, size_t sz); -void vgicWriteGicdRegister(u32 val, size_t offset, size_t sz); -u32 vgicReadGicdRegister(size_t offset, size_t sz); - -void handleVgicdMmio(ExceptionStackFrame *frame, DataAbortIss dabtIss, size_t offset); - -void vgicInit(void); -void vgicUpdateState(void); -void vgicMaintenanceInterruptHandler(void); -void vgicEnqueuePhysicalIrq(u16 irqId);