diff --git a/source/lib/sysdep/numa.h b/source/lib/sysdep/numa.h deleted file mode 100644 index 87e816d3f1..0000000000 --- a/source/lib/sysdep/numa.h +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright (C) 2025 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef INCLUDED_NUMA -#define INCLUDED_NUMA - -#include -#include - -//----------------------------------------------------------------------------- -// node topology - -/** - * @return number of NUMA "nodes" (i.e. groups of CPUs with local memory). - **/ -size_t numa_NumNodes(); - -/** - * @param processor - * @return node number (zero-based) to which \ belongs. - **/ -size_t numa_NodeFromProcessor(size_t processor); - -/** - * @param node - * @return bit-mask of all processors constituting \. - **/ -uintptr_t numa_ProcessorMaskFromNode(size_t node); - - -//----------------------------------------------------------------------------- -// memory - - -/** - * @param node - * @return bytes of memory available for allocation on \. - **/ -size_t numa_AvailableMemory(size_t node); - -/** - * @return the ratio between maximum and minimum times that one processor - * from each node required to fill a globally allocated array. - * in other words, this is the maximum slowdown for NUMA-oblivious - * memory accesses. Microsoft guidelines require it to be <= 3. - **/ -double numa_Factor(); - -/** - * @return an indication of whether memory pages are node-interleaved. - * - * note: this requires ACPI access, which may not be available on - * least-permission accounts. the default is to return false so as - * not to cause callers to panic and trigger performance warnings. - **/ -bool numa_IsMemoryInterleaved(); - -#endif // #ifndef INCLUDED_NUMA diff --git a/source/lib/sysdep/os/unix/unuma.cpp b/source/lib/sysdep/os/unix/unuma.cpp deleted file mode 100644 index 80085cfc79..0000000000 --- a/source/lib/sysdep/os/unix/unuma.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright (C) 2025 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "precompiled.h" - -#include "lib/sysdep/numa.h" - -#include "lib/bits.h" -#include "lib/debug.h" -#include "lib/sysdep/os_cpu.h" - -#include -#include - -size_t numa_NumNodes() -{ - return 1; -} - -size_t numa_NodeFromProcessor(size_t /*processor*/) -{ - return 0; -} - -uintptr_t numa_ProcessorMaskFromNode(size_t node) -{ - ENSURE(node == 0); - return bit_mask(os_cpu_NumProcessors()); -} - -size_t numa_AvailableMemory(size_t node) -{ - ENSURE(node == 0); - return os_cpu_MemoryAvailable(); -} - -double numa_Factor() -{ - return 1.0; -} - -bool numa_IsMemoryInterleaved() -{ - return false; -} diff --git a/source/lib/sysdep/os/unix/unuma.h b/source/lib/sysdep/os/unix/unuma.h deleted file mode 100644 index 25d81dc7f7..0000000000 --- a/source/lib/sysdep/os/unix/unuma.h +++ /dev/null @@ -1,26 +0,0 @@ -/* Copyright (C) 2010 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef INCLUDED_UNUMA -#define INCLUDED_UNUMA - -#endif // #ifndef INCLUDED_UNUMA diff --git a/source/lib/sysdep/os/win/wnuma.cpp b/source/lib/sysdep/os/win/wnuma.cpp deleted file mode 100644 index 6a4365d2bc..0000000000 --- a/source/lib/sysdep/os/win/wnuma.cpp +++ /dev/null @@ -1,497 +0,0 @@ -/* Copyright (C) 2026 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "precompiled.h" -#include "lib/sysdep/numa.h" - -#include "lib/bits.h" -#include "lib/alignment.h" -#include "lib/lib.h" -#include "lib/timer.h" -#include "lib/module_init.h" -#include "lib/sysdep/vm.h" -#include "lib/sysdep/os_cpu.h" -#include "lib/sysdep/os/win/acpi.h" -#include "lib/sysdep/os/win/win.h" -#include "lib/sysdep/os/win/wcpu.h" - -#include -#include -#include -#include - -#if ARCH_X86_X64 -#include "lib/sysdep/arch/x86_x64/apic.h" // ProcessorFromApicId -#endif - - -//----------------------------------------------------------------------------- -// nodes - -struct Node // POD -{ - // (Windows doesn't guarantee node numbers are contiguous, so - // we associate them with contiguous indices in nodes[]) - UCHAR nodeNumber; - - std::uint32_t proximityDomainNumber; - uintptr_t processorMask; -}; - -static Node nodes[os_cpu_MaxProcessors]; -static size_t numNodes; - -static Node* AddNode() -{ - ENSURE(numNodes < ARRAY_SIZE(nodes)); - return &nodes[numNodes++]; -} - -static Node* FindNodeWithProcessorMask(uintptr_t processorMask) -{ - for(size_t node = 0; node < numNodes; node++) - { - if(nodes[node].processorMask == processorMask) - return &nodes[node]; - } - - return 0; -} - -static Node* FindNodeWithProcessor(size_t processor) -{ - for(size_t node = 0; node < numNodes; node++) - { - if(IsBitSet(nodes[node].processorMask, processor)) - return &nodes[node]; - } - - return 0; -} - - -//----------------------------------------------------------------------------- -// Windows topology - -static UCHAR HighestNodeNumber() -{ - ULONG highestNodeNumber{0}; - const BOOL ok = GetNumaHighestNodeNumber(&highestNodeNumber); - WARN_IF_FALSE(ok); - return (UCHAR)highestNodeNumber; -} - -static void PopulateNodes() -{ - DWORD_PTR processAffinity, systemAffinity; - { - const BOOL ok = GetProcessAffinityMask(GetCurrentProcess(), &processAffinity, &systemAffinity); - WARN_IF_FALSE(ok); - } - ENSURE(std::popcount(processAffinity) <= std::popcount(systemAffinity)); - - for(UCHAR nodeNumber = 0; nodeNumber <= HighestNodeNumber(); nodeNumber++) - { - ULONGLONG affinity{0}; - { - const BOOL ok = GetNumaNodeProcessorMask(nodeNumber, &affinity); - WARN_IF_FALSE(ok); - } - if(!affinity) - continue; // empty node, skip - - Node* node = AddNode(); - node->nodeNumber = nodeNumber; - node->processorMask = wcpu_ProcessorMaskFromAffinity(processAffinity, (DWORD_PTR)affinity); - } -} - - -//----------------------------------------------------------------------------- -// ACPI SRAT topology - -#if ARCH_X86_X64 - -#pragma pack(push, 1) - -// fields common to Affinity* structures -struct AffinityHeader -{ - std::uint8_t type; - std::uint8_t length; // size [bytes], including this header -}; - -struct AffinityAPIC -{ - static const std::uint8_t type = 0; - - AffinityHeader header; - std::uint8_t proximityDomainNumber0; - std::uint8_t apicId; - std::uint32_t flags; - std::uint8_t sapicId; - std::uint8_t proximityDomainNumber123[3]; - std::uint32_t clockDomain; - - std::uint32_t ProximityDomainNumber() const - { - // (this is the apparent result of backwards compatibility, ugh.) - std::uint32_t proximityDomainNumber; - memcpy(&proximityDomainNumber, &proximityDomainNumber123[0]-1, sizeof(proximityDomainNumber)); - proximityDomainNumber &= ~0xFF; - proximityDomainNumber |= proximityDomainNumber0; - return proximityDomainNumber; - } -}; - -struct AffinityMemory -{ - static const std::uint8_t type = 1; - - AffinityHeader header; - std::uint32_t proximityDomainNumber; - std::uint16_t reserved1; - std::uint64_t baseAddress; - std::uint64_t length; - std::uint32_t reserved2; - std::uint32_t flags; - std::uint64_t reserved3; -}; - -// AffinityX2APIC omitted, since the APIC ID is sufficient for our purposes - -// Static Resource Affinity Table -struct SRAT -{ - AcpiTable header; - std::uint32_t reserved1; - std::uint8_t reserved2[8]; - AffinityHeader affinities[1]; -}; - -#pragma pack(pop) - -template -static const Affinity* DynamicCastFromHeader(const AffinityHeader* header) -{ - if(header->type != Affinity::type) - return 0; - - // sanity check: ensure no padding was inserted - ENSURE(header->length == sizeof(Affinity)); - - const Affinity* affinity = (const Affinity*)header; - if(!IsBitSet(affinity->flags, 0)) // not enabled - return 0; - - return affinity; -} - -struct ProximityDomain -{ - uintptr_t processorMask; - // (AffinityMemory's fields are not currently needed) -}; - -typedef std::map ProximityDomains; - -static ProximityDomains ExtractProximityDomainsFromSRAT(const SRAT* srat) -{ - ProximityDomains proximityDomains; - - for(const AffinityHeader* header = srat->affinities; - header < (const AffinityHeader*)(uintptr_t(srat)+srat->header.size); - header = (const AffinityHeader*)(uintptr_t(header) + header->length)) - { - const AffinityAPIC* affinityAPIC = DynamicCastFromHeader(header); - if(!affinityAPIC) - continue; - - if(!IsProcessorKnown(affinityAPIC->apicId)) - { - debug_printf("Processor with APIC ID %d found in SRAT but not known to CPUID.", affinityAPIC->apicId); - continue; - } - - const size_t processor = ProcessorFromApicId(affinityAPIC->apicId); - const std::uint32_t proximityDomainNumber = affinityAPIC->ProximityDomainNumber(); - ProximityDomain& proximityDomain = proximityDomains[proximityDomainNumber]; - proximityDomain.processorMask |= Bit(processor); - } - - return proximityDomains; -} - -static void PopulateNodesFromProximityDomains(const ProximityDomains& proximityDomains) -{ - for(ProximityDomains::const_iterator it = proximityDomains.begin(); it != proximityDomains.end(); ++it) - { - const std::uint32_t proximityDomainNumber = it->first; - const ProximityDomain& proximityDomain = it->second; - - Node* node = FindNodeWithProcessorMask(proximityDomain.processorMask); - if(!node) - node = AddNode(); - // (we don't know Windows' nodeNumber; it has hopefully already been set) - node->proximityDomainNumber = proximityDomainNumber; - node->processorMask = proximityDomain.processorMask; - } -} - -#endif // #if ARCH_X86_X64 - - -//----------------------------------------------------------------------------- - -static ModuleInitState initState{ 0 }; - -static Status InitTopology() -{ - PopulateNodes(); - -#if ARCH_X86_X64 - const SRAT* srat = (const SRAT*)acpi_GetTable("SRAT"); - if(srat && AreApicIdsReliable()) - { - const ProximityDomains proximityDomains = ExtractProximityDomainsFromSRAT(srat); - PopulateNodesFromProximityDomains(proximityDomains); - } -#endif - - // neither OS nor ACPI information is available - if(numNodes == 0) - { - // add dummy node that contains all system processors - Node* node = AddNode(); - node->nodeNumber = 0; - node->proximityDomainNumber = 0; - node->processorMask = os_cpu_ProcessorMask(); - } - - return INFO::OK; -} - -size_t numa_NumNodes() -{ - std::ignore = ModuleInit(&initState, InitTopology); - return numNodes; -} - -size_t numa_NodeFromProcessor(size_t processor) -{ - std::ignore = ModuleInit(&initState, InitTopology); - ENSURE(processor < os_cpu_NumProcessors()); - Node* node = FindNodeWithProcessor(processor); - ENSURE(node); - return nodes-node; -} - -uintptr_t numa_ProcessorMaskFromNode(size_t node) -{ - std::ignore = ModuleInit(&initState, InitTopology); - ENSURE(node < numNodes); - return nodes[node].processorMask; -} - -static UCHAR NodeNumberFromNode(size_t node) -{ - std::ignore = ModuleInit(&initState, InitTopology); - ENSURE(node < numa_NumNodes()); - return nodes[node].nodeNumber; -} - - -//----------------------------------------------------------------------------- -// memory info - -size_t numa_AvailableMemory(size_t node) -{ - // note: it is said that GetNumaAvailableMemoryNode sometimes incorrectly - // reports zero bytes. the actual cause may however be unexpected - // RAM configuration, e.g. not all slots filled. - const UCHAR nodeNumber = NodeNumberFromNode(node); - ULONGLONG availableBytes{0}; - const BOOL ok = GetNumaAvailableMemoryNode(nodeNumber, &availableBytes); - WARN_IF_FALSE(ok); - const size_t availableMiB = size_t(availableBytes / MiB); - return availableMiB; -} - - -#pragma pack(push, 1) - -// ACPI System Locality Information Table -// (System Locality == Proximity Domain) -struct SLIT -{ - AcpiTable header; - std::uint64_t numSystemLocalities; - std::uint8_t entries[1]; // numSystemLocalities*numSystemLocalities entries -}; - -#pragma pack(pop) - -static double ReadRelativeDistanceFromSLIT(const SLIT* slit) -{ - const size_t n = slit->numSystemLocalities; - ENSURE(slit->header.size == sizeof(SLIT)-sizeof(slit->entries)+n*n); - // diagonals are specified to be 10 - for(size_t i = 0; i < n; i++) - ENSURE(slit->entries[i*n+i] == 10); - // entries = relativeDistance * 10 - return *std::max_element(slit->entries, slit->entries+n*n) / 10.0; -} - -// @return ratio between max/min time required to access one node's -// memory from each processor. -static double MeasureRelativeDistance() -{ - const size_t size = 32*MiB; - void* mem = vm::Allocate(size); - - const uintptr_t previousProcessorMask = os_cpu_SetThreadAffinityMask(os_cpu_ProcessorMask()); - - double minTime = 1e10, maxTime = 0.0; - for(size_t node = 0; node < numa_NumNodes(); node++) - { - const uintptr_t processorMask = numa_ProcessorMaskFromNode(node); - os_cpu_SetThreadAffinityMask(processorMask); - - const double startTime = timer_Time(); - memset(mem, 0, size); - const double elapsedTime = timer_Time() - startTime; - - minTime = std::min(minTime, elapsedTime); - maxTime = std::max(maxTime, elapsedTime); - } - - std::ignore = os_cpu_SetThreadAffinityMask(previousProcessorMask); - - vm::Free(mem, size); - - return maxTime / minTime; -} - -static double relativeDistance; - -static Status InitRelativeDistance() -{ - // early-out for non-NUMA systems (saves some time) - if(numa_NumNodes() == 1) - { - relativeDistance = 1.0; - return INFO::OK; - } - - // trust values reported by the BIOS, if available - const SLIT* slit = (const SLIT*)acpi_GetTable("SLIT"); - if(slit) - relativeDistance = ReadRelativeDistanceFromSLIT(slit); - else - relativeDistance = MeasureRelativeDistance(); - - ENSURE(relativeDistance >= 1.0); - ENSURE(relativeDistance <= 4.0); - return INFO::OK; -} - -double numa_Factor() -{ - static ModuleInitState _initState{ 0 }; - std::ignore = ModuleInit(&_initState, InitRelativeDistance); - return relativeDistance; -} - - -static bool IsMemoryInterleaved() -{ - if(numa_NumNodes() == 1) - return false; - - if(!acpi_GetTable("FACP")) // no ACPI tables available - return false; // indeterminate, assume not interleaved - - if(acpi_GetTable("SRAT")) // present iff not interleaved - return false; - - return true; -} - -static bool isMemoryInterleaved; - -static Status InitMemoryInterleaved() -{ - isMemoryInterleaved = IsMemoryInterleaved(); - return INFO::OK; -} - -bool numa_IsMemoryInterleaved() -{ - static ModuleInitState _initState{ 0 }; - std::ignore = ModuleInit(&_initState, InitMemoryInterleaved); - return isMemoryInterleaved; -} - - -//----------------------------------------------------------------------------- - -#if 0 - -static bool VerifyPages([[maybe_unused]] void* mem, [[maybe_unused]] size_t size, - [[maybe_unused]] size_t pageSize, [[maybe_unused]] size_t node) -{ -#if WINVER >= 0x600 - size_t largePageSize = os_cpu_LargePageSize(); - ENSURE(largePageSize != 0); // this value is needed for later - - // retrieve attributes of all pages constituting mem - const size_t numPages = (size + pageSize-1) / pageSize; - PSAPI_WORKING_SET_EX_INFORMATION* wsi = new PSAPI_WORKING_SET_EX_INFORMATION[numPages]; - for(size_t i = 0; i < numPages; i++) - wsi[i].VirtualAddress = static_cast(mem) + i*pageSize; - QueryWorkingSetEx(GetCurrentProcess(), wsi, DWORD(sizeof(PSAPI_WORKING_SET_EX_INFORMATION)*numPages)); - - // ensure each is valid and allocated on the correct node - for(size_t i = 0; i < numPages; i++) - { - const PSAPI_WORKING_SET_EX_BLOCK& attributes = wsi[i].VirtualAttributes; - if(!attributes.Valid) - return false; - if((attributes.LargePage != 0) != (pageSize == largePageSize)) - { - debug_printf("NUMA: is not a large page\n"); - return false; - } - if(attributes.Node != node) - { - debug_printf("NUMA: allocated from remote node\n"); - return false; - } - } - - delete[] wsi; -#endif - - return true; -} - -#endif diff --git a/source/lib/sysdep/os/win/wvm.cpp b/source/lib/sysdep/os/win/wvm.cpp index 02bd69524a..074d67ba94 100644 --- a/source/lib/sysdep/os/win/wvm.cpp +++ b/source/lib/sysdep/os/win/wvm.cpp @@ -31,7 +31,6 @@ #include "lib/alignment.h" // CACHE_ALIGNED #include "lib/bits.h" // round_down #include "lib/module_init.h" -#include "lib/sysdep/numa.h" #include "lib/sysdep/os/win/wutil.h" #include "lib/timer.h" @@ -89,9 +88,6 @@ void DumpStatistics() debug_printf("%d commits (%d, i.e. %d%% of them via large pages)\n", totalCommits, largePageCommits, largePageRatio); if(processorsWithNoCommits != 0) debug_printf(" processors with no commits: %x\n", processorsWithNoCommits); - - if(numa_NumNodes() > 1) - debug_printf("NUMA factor: %.2f\n", numa_Factor()); }