Atmosphere/libraries/libexosphere/source/hw/hw_cache.arch.arm.cpp

/*
 * Copyright (c) Atmosphère-NX
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include <exosphere.hpp>

namespace ams::hw::arch::arm {

#ifdef __BPMP__

    namespace {

        constexpr inline uintptr_t AVP_CACHE = AVP_CACHE_ADDR(0);

        ALWAYS_INLINE bool IsLargeBuffer(size_t size) {
            /* From TRM: For very large physical buffers or when the full cache needs to be cleared, */
            /*           software should simply loop over all lines in all ways and run the *_LINE command on each of them. */
            return size >= DataCacheSize / 4;
        }

        ALWAYS_INLINE bool IsCacheEnabled() {
            return reg::HasValue(AVP_CACHE + AVP_CACHE_CONFIG, AVP_CACHE_REG_BITS_ENUM(CONFIG_ENABLE_CACHE, TRUE));
        }

        void DoPhyCacheOperation(AVP_CACHE_MAINT_OPCODE op, uintptr_t addr) {
            /* Clear maintenance done. */
            reg::Write(AVP_CACHE + AVP_CACHE_INT_CLEAR, AVP_CACHE_REG_BITS_ENUM(INT_CLEAR_MAINTENANCE_DONE, TRUE));

            /* Write maintenance address. */
            reg::Write(AVP_CACHE + AVP_CACHE_MAINT_0, addr);

            /* Write maintenance request. */
            reg::Write(AVP_CACHE + AVP_CACHE_MAINT_2, AVP_CACHE_REG_BITS_VALUE(MAINT_2_WAY_BITMAP, 0x0),
                                                      AVP_CACHE_REG_BITS_VALUE(MAINT_2_OPCODE,      op));

            /* Wait for maintenance to be done. */
            while (!reg::HasValue(AVP_CACHE + AVP_CACHE_INT_RAW_EVENT, AVP_CACHE_REG_BITS_ENUM(INT_RAW_EVENT_MAINTENANCE_DONE, TRUE))) {
                /* ... */
            }

            /* Clear raw event. */
            reg::Write(AVP_CACHE + AVP_CACHE_INT_CLEAR, reg::Read(AVP_CACHE + AVP_CACHE_INT_RAW_EVENT));
        }

        void DoEntireCacheOperation(AVP_CACHE_MAINT_OPCODE op) {
            /* Clear maintenance done. */
            reg::Write(AVP_CACHE + AVP_CACHE_INT_CLEAR, AVP_CACHE_REG_BITS_ENUM(INT_CLEAR_MAINTENANCE_DONE, TRUE));

            /* Write maintenance request. */
            reg::Write(AVP_CACHE + AVP_CACHE_MAINT_2, AVP_CACHE_REG_BITS_VALUE(MAINT_2_WAY_BITMAP, 0xF),
                                                      AVP_CACHE_REG_BITS_VALUE(MAINT_2_OPCODE,      op));

            /* Wait for maintenance to be done. */
            while (!reg::HasValue(AVP_CACHE + AVP_CACHE_INT_RAW_EVENT, AVP_CACHE_REG_BITS_ENUM(INT_RAW_EVENT_MAINTENANCE_DONE, TRUE))) {
                /* ... */
            }

            /* Clear raw event. */
            reg::Write(AVP_CACHE + AVP_CACHE_INT_CLEAR, reg::Read(AVP_CACHE + AVP_CACHE_INT_RAW_EVENT));
        }

    }

    #define REQUIRE_CACHE_ENABLED()                \
        do {                                       \
            if (AMS_UNLIKELY(!IsCacheEnabled())) { \
                return;                            \
            }                                      \
        } while (false)                            \


    #define REQUIRE_CACHE_DISABLED()              \
        do {                                      \
            if (AMS_UNLIKELY(IsCacheEnabled())) { \
                return;                           \
            }                                     \
        } while (false)                           \

    void InitializeDataCache() {
        REQUIRE_CACHE_DISABLED();

        /* Issue init mmu command. */
        reg::Write(AVP_CACHE + AVP_CACHE_MMU_CMD, AVP_CACHE_REG_BITS_ENUM(MMU_CMD_CMD, INIT));

        /* Set mmu fallback entry as RWX, uncached. */
        reg::Write(AVP_CACHE + AVP_CACHE_MMU_FALLBACK_ENTRY, AVP_CACHE_REG_BITS_ENUM(MMU_FALLBACK_ENTRY_WR_ENA,   ENABLE),
                                                             AVP_CACHE_REG_BITS_ENUM(MMU_FALLBACK_ENTRY_RD_ENA,   ENABLE),
                                                             AVP_CACHE_REG_BITS_ENUM(MMU_FALLBACK_ENTRY_EXE_ENA,  ENABLE),
                                                             AVP_CACHE_REG_BITS_ENUM(MMU_FALLBACK_ENTRY_CACHED,  DISABLE));

        /* Set mmu cfg. */
        reg::Write(AVP_CACHE + AVP_CACHE_MMU_CFG, AVP_CACHE_REG_BITS_ENUM(MMU_CFG_CLR_ABORT,                    NOP),
                                                  AVP_CACHE_REG_BITS_ENUM(MMU_CFG_ABORT_MODE,            STORE_LAST),
                                                  AVP_CACHE_REG_BITS_ENUM(MMU_CFG_SEQ_CHECK_ALL_ENTRIES,    DISABLE),
                                                  AVP_CACHE_REG_BITS_ENUM(MMU_CFG_TLB_ENA,                   ENABLE),
                                                  AVP_CACHE_REG_BITS_ENUM(MMU_CFG_SEQ_ENA,                   ENABLE),
                                                  AVP_CACHE_REG_BITS_ENUM(MMU_CFG_BLOCK_MAIN_ENTRY_WR,      DISABLE));

        /* Initialize mmu entries. */
        {
            /* Clear shadow copy mask. */
            reg::Write(AVP_CACHE + AVP_CACHE_MMU_SHADOW_COPY_MASK_0, 0);

            /* Add DRAM as index 0, RWX/Cached. */
            {
                reg::Write(AVP_CACHE + AVP_CACHE_MMU_SHADOW_ENTRY_0_MIN_ADDR, 0x80000000);
                reg::Write(AVP_CACHE + AVP_CACHE_MMU_SHADOW_ENTRY_0_MAX_ADDR, util::AlignDown(0xFFFFFFFF, DataCacheLineSize));

                reg::Write(AVP_CACHE + AVP_CACHE_MMU_SHADOW_ENTRY_0_CFG, AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_WR_ENA,  ENABLE),
                                                                         AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_RD_ENA,  ENABLE),
                                                                         AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_EXE_ENA, ENABLE),
                                                                         AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_CACHED,  ENABLE));

                reg::SetBits(AVP_CACHE + AVP_CACHE_MMU_SHADOW_COPY_MASK_0, (1 << 0));
            }

            /* Add IRAM as index 1, RWX/Cached. */
            {
                reg::Write(AVP_CACHE + AVP_CACHE_MMU_SHADOW_ENTRY_1_MIN_ADDR, 0x40000000);
                reg::Write(AVP_CACHE + AVP_CACHE_MMU_SHADOW_ENTRY_1_MAX_ADDR, util::AlignDown(0x4003FFFF, DataCacheLineSize));

                reg::Write(AVP_CACHE + AVP_CACHE_MMU_SHADOW_ENTRY_1_CFG, AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_WR_ENA,  ENABLE),
                                                                         AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_RD_ENA,  ENABLE),
                                                                         AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_EXE_ENA, ENABLE),
                                                                         AVP_CACHE_REG_BITS_ENUM(SHADOW_ENTRY_CFG_CACHED,  ENABLE));

                reg::SetBits(AVP_CACHE + AVP_CACHE_MMU_SHADOW_COPY_MASK_0, (1 << 1));
            }

            /* Issue copy shadow mmu command. */
            reg::Write(AVP_CACHE + AVP_CACHE_MMU_CMD, AVP_CACHE_REG_BITS_ENUM(MMU_CMD_CMD, COPY_SHADOW));
        }

        /* Invalidate entire cache. */
        DoEntireCacheOperation(AVP_CACHE_MAINT_OPCODE_INVALID_WAY);

        /* Enable the cache. */
        reg::Write(AVP_CACHE + AVP_CACHE_CONFIG, AVP_CACHE_REG_BITS_ENUM(CONFIG_ENABLE_CACHE,                 TRUE),
                                                 AVP_CACHE_REG_BITS_ENUM(CONFIG_FORCE_WRITE_THROUGH,          TRUE),
                                                 AVP_CACHE_REG_BITS_ENUM(CONFIG_MMU_TAG_MODE,             PARALLEL),
                                                 AVP_CACHE_REG_BITS_ENUM(CONFIG_TAG_CHECK_ABORT_ON_ERROR,     TRUE));

        /* Invalidate entire cache again (WAR for hardware bug). */
        DoEntireCacheOperation(AVP_CACHE_MAINT_OPCODE_INVALID_WAY);
    }

    void FinalizeDataCache() {
        REQUIRE_CACHE_ENABLED();

        /* Flush entire data cache. */
        FlushEntireDataCache();

        /* Disable cache. */
        reg::Write(AVP_CACHE + AVP_CACHE_CONFIG, AVP_CACHE_REG_BITS_ENUM(CONFIG_ENABLE_CACHE, FALSE));
    }

    void InvalidateEntireDataCache() {
        REQUIRE_CACHE_ENABLED();

        DoEntireCacheOperation(AVP_CACHE_MAINT_OPCODE_INVALID_WAY);
    }

    void StoreEntireDataCache() {
        REQUIRE_CACHE_ENABLED();

        DoEntireCacheOperation(AVP_CACHE_MAINT_OPCODE_CLEAN_WAY);
    }

    void FlushEntireDataCache() {
        REQUIRE_CACHE_ENABLED();

        DoEntireCacheOperation(AVP_CACHE_MAINT_OPCODE_CLEAN_INVALID_WAY);
    }

    void InvalidateDataCacheLine(void *ptr) {
        /* NOTE: Don't check cache enabled as an optimization, as only direct caller will be InvalidateDataCache(). */
        /* REQUIRE_CACHE_ENABLED(); */

        DoPhyCacheOperation(AVP_CACHE_MAINT_OPCODE_INVALID_PHY, reinterpret_cast<uintptr_t>(ptr));
    }

    void StoreDataCacheLine(void *ptr) {
        /* NOTE: Don't check cache enabled as an optimization, as only direct caller will be FlushDataCache(). */
        /* REQUIRE_CACHE_ENABLED(); */

        DoPhyCacheOperation(AVP_CACHE_MAINT_OPCODE_CLEAN_PHY, reinterpret_cast<uintptr_t>(ptr));
    }

    void FlushDataCacheLine(void *ptr) {
        /* NOTE: Don't check cache enabled as an optimization, as only direct caller will be FlushDataCache(). */
        /* REQUIRE_CACHE_ENABLED(); */

        DoPhyCacheOperation(AVP_CACHE_MAINT_OPCODE_CLEAN_INVALID_PHY, reinterpret_cast<uintptr_t>(ptr));
    }

    void InvalidateDataCache(void *ptr, size_t size) {
        REQUIRE_CACHE_ENABLED();

        if (IsLargeBuffer(size)) {
            InvalidateEntireDataCache();
        } else {
            const uintptr_t start = reinterpret_cast<uintptr_t>(ptr);
            const uintptr_t end   = util::AlignUp(start + size, hw::DataCacheLineSize);

            for (uintptr_t cur = start; cur < end; cur += hw::DataCacheLineSize) {
                InvalidateDataCacheLine(reinterpret_cast<void *>(cur));
            }
        }
    }

    void StoreDataCache(const void *ptr, size_t size) {
        REQUIRE_CACHE_ENABLED();

        if (IsLargeBuffer(size)) {
            StoreEntireDataCache();
        } else {
            const uintptr_t start = reinterpret_cast<uintptr_t>(ptr);
            const uintptr_t end   = util::AlignUp(start + size, hw::DataCacheLineSize);

            for (uintptr_t cur = start; cur < end; cur += hw::DataCacheLineSize) {
                StoreDataCacheLine(reinterpret_cast<void *>(cur));
            }
        }
    }

    void FlushDataCache(const void *ptr, size_t size) {
        REQUIRE_CACHE_ENABLED();

        if (IsLargeBuffer(size)) {
            FlushEntireDataCache();
        } else {
            const uintptr_t start = reinterpret_cast<uintptr_t>(ptr);
            const uintptr_t end   = util::AlignUp(start + size, hw::DataCacheLineSize);

            for (uintptr_t cur = start; cur < end; cur += hw::DataCacheLineSize) {
                FlushDataCacheLine(reinterpret_cast<void *>(cur));
            }
        }
    }
#endif

}