From 881d3cebf49ac2fe8e8ff773d28f7a9b9d0bdf69 Mon Sep 17 00:00:00 2001 From: janwas Date: Sun, 21 Aug 2011 11:00:09 +0000 Subject: [PATCH] lay groundwork for more efficient and flexible allocators. add new sysdep/vm that provides access to additional features on Windows (large pages, autocommit). add Pool/Arena allocators that avoid overhead and support arbitrary storage (not just the expensive virtual memory allocator in DynArray) This was SVN commit r10051. --- source/lib/alignment.h | 2 +- source/lib/allocators/allocator_adapters.h | 95 +++ source/lib/allocators/allocator_policies.h | 354 ++++++++++++ source/lib/allocators/arena.h | 82 +++ source/lib/allocators/dynarray.cpp | 38 +- source/lib/allocators/dynarray.h | 17 - source/lib/allocators/overrun_protector.h | 10 +- source/lib/allocators/pool.cpp | 52 +- source/lib/allocators/pool.h | 113 ++-- source/lib/allocators/tests/test_allocators.h | 1 - source/lib/allocators/unique_range.cpp | 18 +- source/lib/allocators/unique_range.h | 4 + source/lib/bits.h | 4 + source/lib/debug.cpp | 6 +- source/lib/debug.h | 4 +- source/lib/pch/pch_warnings.h | 1 + source/lib/sysdep/os/unix/uvm.cpp | 136 +++++ source/lib/sysdep/os/win/wnuma.cpp | 137 ++--- source/lib/sysdep/os/win/wposix/wmman.cpp | 29 +- source/lib/sysdep/os/win/wposix/wmman.h | 5 +- source/lib/sysdep/os/win/wvm.cpp | 539 ++++++++++++++++++ source/lib/sysdep/vm.h | 154 +++++ 22 files changed, 1604 insertions(+), 197 deletions(-) create mode 100644 source/lib/allocators/allocator_adapters.h create mode 100644 source/lib/allocators/allocator_policies.h create mode 100644 source/lib/allocators/arena.h create mode 100644 source/lib/sysdep/os/unix/uvm.cpp create mode 100644 source/lib/sysdep/os/win/wvm.cpp create mode 100644 source/lib/sysdep/vm.h diff --git a/source/lib/alignment.h b/source/lib/alignment.h index b4947c5e0a..d122efbc1b 100644 --- a/source/lib/alignment.h +++ b/source/lib/alignment.h @@ -65,7 +65,7 @@ static const size_t largePageSize = 0x200000; // 2 MB // misc // -static const size_t allocationAlignment = ARCH_AMD64? 16 : 8; +static const size_t allocationAlignment = 16; static const size_t KiB = size_t(1) << 10; static const size_t MiB = size_t(1) << 20; diff --git a/source/lib/allocators/allocator_adapters.h b/source/lib/allocators/allocator_adapters.h new file mode 100644 index 0000000000..117dab88c5 --- /dev/null +++ b/source/lib/allocators/allocator_adapters.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2011 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * adapters for allocators; provides a minimal subset of the + * STL allocator interface. + */ + +#ifndef ALLOCATOR_ADAPTERS +#define ALLOCATOR_ADAPTERS + +#include + +#include "lib/sysdep/rtl.h" +#include "lib/sysdep/vm.h" + +// NB: STL allocators are parameterized on the object type and indicate +// the number of elements to [de]allocate. however, these adapters are +// only used for allocating storage and receive the number of bytes. + +struct Allocator_Heap +{ + void* allocate(size_t size) + { + return malloc(size); + } + + void deallocate(void* p, size_t UNUSED(size)) + { + return free(p); + } +}; + +template +struct Allocator_Aligned +{ + void* allocate(size_t size) + { + return rtl_AllocateAligned(size, alignment); + } + + void deallocate(void* p, size_t UNUSED(size)) + { + return rtl_FreeAligned(p); + } +}; + +template +struct Allocator_VM +{ + void* allocate(size_t size) + { + return vm::Allocate(size, pageType, prot); + } + + void deallocate(void* p, size_t size) + { + vm::Free(p, size); + } +}; + +template +struct Allocator_AddressSpace +{ + void* allocate(size_t size) + { + return vm::ReserveAddressSpace(size, commitSize, pageType, prot); + } + + void deallocate(void* p, size_t size) + { + vm::ReleaseAddressSpace(p, size); + } +}; + +#endif // #ifndef ALLOCATOR_ADAPTERS diff --git a/source/lib/allocators/allocator_policies.h b/source/lib/allocators/allocator_policies.h new file mode 100644 index 0000000000..7f73c535e0 --- /dev/null +++ b/source/lib/allocators/allocator_policies.h @@ -0,0 +1,354 @@ +/* Copyright (c) 2011 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * policy class templates for allocators. + */ + +#ifndef ALLOCATOR_POLICIES +#define ALLOCATOR_POLICIES + +#include "lib/alignment.h" // pageSize +#include "lib/allocators/allocator_adapters.h" +#include "lib/allocators/freelist.h" + + +namespace Allocators { + + +//----------------------------------------------------------------------------- +// Growth + +// O(N) allocations, O(1) wasted space. +template +struct Growth_Linear +{ + size_t operator()(size_t oldSize) const + { + return oldSize + increment; + } +}; + +// O(log r) allocations, O(N) wasted space. NB: the common choice of +// expansion factor r = 2 (e.g. in the GCC STL) prevents +// Storage_Reallocate from reusing previous memory blocks, +// thus constantly growing the heap and decreasing locality. +// Alexandrescu [C++ and Beyond 2010] recommends r < 33/25. +// we approximate this with a power of two divisor to allow shifting. +// C++ does allow reference-to-float template parameters, but +// integer arithmetic is expected to be faster. +// (Storage_Commit should use 2:1 because it is cheaper to +// compute and retains power-of-two sizes.) +template +struct Growth_Exponential +{ + size_t operator()(size_t oldSize) const + { + const size_t product = oldSize * multiplier; + + // detect overflow, but allow equality in case oldSize = 0, + // which isn't a problem because Storage_Commit::Expand + // raises it to requiredCapacity. + ASSERT(product >= oldSize); + + return product / divisor; + } +}; + + +//----------------------------------------------------------------------------- +// Storage + +// a contiguous region of memory (not just an "array", because +// allocators such as Arena append variable-sized intervals). +// +// we don't store smart pointers because storage usually doesn't need +// to be copied, and ICC 11 sometimes wasn't able to inline Address(). +struct Storage +{ + // @return starting address (alignment depends on the allocator). + uintptr_t Address() const; + + // @return size [bytes] of currently accessible memory. + size_t Capacity() const; + + // @return largest possible capacity [bytes]. + size_t MaxCapacity() const; + + // expand Capacity() to at least requiredCapacity (possibly more + // depending on GrowthPolicy). + // @param requiredCapacity > Capacity() + // @return false and leave Capacity() unchanged if expansion failed, + // which is guaranteed to happen if requiredCapacity > MaxCapacity(). + bool Expand(size_t requiredCapacity); +}; + + +// allocate once and refuse subsequent expansion. +template > +class Storage_Fixed +{ + NONCOPYABLE(Storage_Fixed); +public: + Storage_Fixed(size_t size) + : maxCapacity(size) + , storage(allocator.allocate(maxCapacity)) + { + } + + ~Storage_Fixed() + { + allocator.deallocate(storage, maxCapacity); + } + + uintptr_t Address() const + { + return uintptr_t(storage); + } + + size_t Capacity() const + { + return maxCapacity; + } + + size_t MaxCapacity() const + { + return maxCapacity; + } + + bool Expand(size_t UNUSED(requiredCapacity)) + { + return false; + } + +private: + Allocator allocator; + size_t maxCapacity; // must be initialized before storage + void* storage; +}; + + +// unlimited expansion by allocating larger storage and copying. +// (basically equivalent to std::vector, although Growth_Exponential +// is much more cache and allocator-friendly than the GCC STL) +template > +class Storage_Reallocate +{ + NONCOPYABLE(Storage_Reallocate); +public: + Storage_Reallocate(size_t initialCapacity) + : capacity(initialCapacity) + , storage(allocator.allocate(initialCapacity)) + { + } + + ~Storage_Reallocate() + { + allocator.deallocate(storage, capacity); + } + + uintptr_t Address() const + { + return uintptr_t(storage); + } + + size_t Capacity() const + { + return capacity; + } + + size_t MaxCapacity() const + { + return std::numeric_limits::max(); + } + + bool Expand(size_t requiredCapacity) + { + size_t newCapacity = std::max(requiredCapacity, GrowthPolicy()(capacity)); + void* newStorage = allocator.allocate(newCapacity); + if(!newStorage) + return false; + memcpy(newStorage, storage, capacity); + std::swap(capacity, newCapacity); + std::swap(storage, newStorage); + allocator.deallocate(newStorage, newCapacity); // free PREVIOUS storage + return true; + } + +private: + Allocator allocator; + size_t capacity; // must be initialized before storage + void* storage; +}; + + +// expand up to the limit of the allocated address space by +// committing physical memory. this avoids copying and +// reduces wasted physical memory. +template, class GrowthPolicy = Growth_Exponential<2,1> > +class Storage_Commit +{ + NONCOPYABLE(Storage_Commit); +public: + Storage_Commit(size_t maxCapacity_) + : maxCapacity(Align(maxCapacity_)) // see Expand + , storage(allocator.allocate(maxCapacity)) + , capacity(0) + { + } + + ~Storage_Commit() + { + allocator.deallocate(storage, maxCapacity); + } + + uintptr_t Address() const + { + return uintptr_t(storage); + } + + size_t Capacity() const + { + return capacity; + } + + size_t MaxCapacity() const + { + return maxCapacity; + } + + bool Expand(size_t requiredCapacity) + { + size_t newCapacity = std::max(requiredCapacity, GrowthPolicy()(capacity)); + // reduce the number of expensive commits by accurately + // reflecting the actual capacity. this is safe because + // we also round up maxCapacity. + newCapacity = Align(newCapacity); + if(newCapacity > maxCapacity) + return false; + if(!vm::Commit(Address()+capacity, newCapacity-capacity)) + return false; + capacity = newCapacity; + return true; + } + +private: + Allocator allocator; + size_t maxCapacity; // must be initialized before storage + void* storage; + size_t capacity; +}; + + +// implicitly expand up to the limit of the allocated address space by +// committing physical memory when a page is first accessed. +// this is basically equivalent to Storage_Commit with Growth_Linear, +// except that there is no need to call Expand. +template > +class Storage_AutoCommit +{ + NONCOPYABLE(Storage_AutoCommit); +public: + Storage_AutoCommit(size_t maxCapacity_) + : maxCapacity(Align(maxCapacity_)) // match user's expectation + , storage(allocator.allocate(maxCapacity)) + { + vm::BeginOnDemandCommits(); + } + + ~Storage_AutoCommit() + { + vm::EndOnDemandCommits(); + allocator.deallocate(storage, maxCapacity); + } + + uintptr_t Address() const + { + return uintptr_t(storage); + } + + size_t Capacity() const + { + return maxCapacity; + } + + size_t MaxCapacity() const + { + return maxCapacity; + } + + bool Expand(size_t UNUSED(requiredCapacity)) + { + return false; + } + +private: + Allocator allocator; + size_t maxCapacity; // must be initialized before storage + void* storage; +}; + + +// reserve and return a pointer to space at the end of storage, +// expanding it if need be. +// @param end total number of previously reserved bytes; will be +// increased by size if the allocation succeeds. +// @param size [bytes] to reserve. +// @return address of allocated space, or 0 if storage is full +// and cannot expand any further. +template +static inline uintptr_t StorageAppend(Storage& storage, size_t& end, size_t size) +{ + size_t newEnd = end + size; + if(newEnd > storage.Capacity()) + { + if(!storage.Expand(newEnd)) // NB: may change storage.Address() + return 0; + } + + std::swap(end, newEnd); + return storage.Address() + newEnd; +} + + +// invoke operator() on default-constructed instantiations of +// Functor for reasonable combinations of Storage and their parameters. +template class Functor> +static void ForEachStorage() +{ + Functor >()(); + Functor > >()(); + + Functor > >()(); + Functor > >()(); + Functor, Growth_Linear<> > >()(); + Functor, Growth_Exponential<> > >()(); + + Functor, Growth_Linear<> > >()(); + Functor, Growth_Exponential<> > >()(); + + Functor >()(); +} + +} // namespace Allocators + +#endif // #ifndef ALLOCATOR_POLICIES diff --git a/source/lib/allocators/arena.h b/source/lib/allocators/arena.h new file mode 100644 index 0000000000..fe5a323c40 --- /dev/null +++ b/source/lib/allocators/arena.h @@ -0,0 +1,82 @@ +/* Copyright (c) 2010 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * arena allocator (variable-size blocks, no deallocation). + */ + +#ifndef INCLUDED_ALLOCATORS_ARENA +#define INCLUDED_ALLOCATORS_ARENA + +#include "lib/allocators/allocator_policies.h" + +namespace Allocators { + +/** + * allocator design parameters: + * - O(1) allocation; + * - variable-size blocks; + * - support for deallocating all objects; + * - consecutive allocations are back-to-back; + * - no extra alignment nor padding. + **/ +template +class Arena +{ +public: + Arena(size_t maxSize) + : storage(maxSize) + { + DeallocateAll(); + } + + size_t RemainingBytes() const + { + return storage.MaxCapacity() - end; + } + + void* Allocate(size_t size) + { + return (void*)StorageAppend(storage, end, size); + } + + void DeallocateAll() + { + end = 0; + } + + // @return whether the address lies within the previously allocated range. + bool Contains(uintptr_t address) const + { + return (address - storage.Address()) < end; + } + +private: + Storage storage; + size_t end; +}; + +LIB_API void TestArena(); + +} // namespace Allocators + +#endif // #ifndef INCLUDED_ALLOCATORS_ARENA diff --git a/source/lib/allocators/dynarray.cpp b/source/lib/allocators/dynarray.cpp index 40924f4fcd..6e0716e616 100644 --- a/source/lib/allocators/dynarray.cpp +++ b/source/lib/allocators/dynarray.cpp @@ -28,7 +28,7 @@ #include "lib/allocators/dynarray.h" #include "lib/alignment.h" -#include "lib/allocators/page_aligned.h" +#include "lib/sysdep/vm.h" static Status validate_da(DynArray* da) @@ -39,7 +39,6 @@ static Status validate_da(DynArray* da) const size_t max_size_pa = da->max_size_pa; const size_t cur_size = da->cur_size; const size_t pos = da->pos; - const int prot = da->prot; // note: this happens if max_size == 0 // if(debug_IsPointerBogus(base)) @@ -52,8 +51,6 @@ static Status validate_da(DynArray* da) WARN_RETURN(ERR::_4); if(pos > cur_size || pos > max_size_pa) WARN_RETURN(ERR::_5); - if(prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC)) - WARN_RETURN(ERR::_6); return INFO::OK; } @@ -63,17 +60,17 @@ static Status validate_da(DynArray* da) Status da_alloc(DynArray* da, size_t max_size) { + ENSURE(max_size != 0); const size_t max_size_pa = Align(max_size); - u8* p = 0; - if(max_size_pa) // (avoid mmap failure) - RETURN_STATUS_IF_ERR(mem_Reserve(max_size_pa, &p)); + u8* p = (u8*)vm::ReserveAddressSpace(max_size_pa); + if(!p) + return ERR::NO_MEM; // NOWARN (already done in vm) da->base = p; da->max_size_pa = max_size_pa; da->cur_size = 0; da->cur_size_pa = 0; - da->prot = PROT_READ|PROT_WRITE; da->pos = 0; CHECK_DA(da); return INFO::OK; @@ -84,15 +81,11 @@ Status da_free(DynArray* da) { CHECK_DA(da); - u8* p = da->base; - size_t size_pa = da->max_size_pa; + vm::ReleaseAddressSpace(da->base, da->max_size_pa); // wipe out the DynArray for safety - // (must be done here because mem_Release may fail) memset(da, 0, sizeof(*da)); - if(size_pa) - RETURN_STATUS_IF_ERR(mem_Release(p, size_pa)); return INFO::OK; } @@ -113,19 +106,20 @@ Status da_set_size(DynArray* da, size_t new_size) return ERR::LIMIT; // NOWARN u8* end = da->base + cur_size_pa; + bool ok = true; // expanding if(size_delta_pa > 0) - RETURN_STATUS_IF_ERR(mem_Commit(end, size_delta_pa, da->prot)); + ok = vm::Commit(uintptr_t(end), size_delta_pa); // shrinking else if(size_delta_pa < 0) - RETURN_STATUS_IF_ERR(mem_Decommit(end+size_delta_pa, -size_delta_pa)); + ok = vm::Decommit(uintptr_t(end+size_delta_pa), -size_delta_pa); // else: no change in page count, e.g. if going from size=1 to 2 // (we don't want mem_* to have to handle size=0) da->cur_size = new_size; da->cur_size_pa = new_size_pa; CHECK_DA(da); - return INFO::OK; + return ok? INFO::OK : ERR::FAIL; } @@ -138,18 +132,6 @@ Status da_reserve(DynArray* da, size_t size) } -Status da_set_prot(DynArray* da, int prot) -{ - CHECK_DA(da); - - da->prot = prot; - RETURN_STATUS_IF_ERR(mem_Protect(da->base, da->cur_size_pa, prot)); - - CHECK_DA(da); - return INFO::OK; -} - - Status da_append(DynArray* da, const void* data, size_t size) { RETURN_STATUS_IF_ERR(da_reserve(da, size)); diff --git a/source/lib/allocators/dynarray.h b/source/lib/allocators/dynarray.h index d95bcaa9e5..e1af79e801 100644 --- a/source/lib/allocators/dynarray.h +++ b/source/lib/allocators/dynarray.h @@ -43,11 +43,6 @@ struct DynArray size_t cur_size; /// committed size_t cur_size_pa; - /** - * mprotect flags applied to newly committed pages - **/ - int prot; - size_t pos; }; @@ -97,18 +92,6 @@ LIB_API Status da_set_size(DynArray* da, size_t new_size); **/ LIB_API Status da_reserve(DynArray* da, size_t size); -/** - * change access rights of the array memory. - * - * used to implement write-protection. affects the currently committed - * pages as well as all subsequently added pages. - * - * @param da DynArray. - * @param prot a combination of the PROT_* values used with mprotect. - * @return Status. - **/ -LIB_API Status da_set_prot(DynArray* da, int prot); - /** * "write" to array, i.e. copy from the given buffer. * diff --git a/source/lib/allocators/overrun_protector.h b/source/lib/allocators/overrun_protector.h index d526fa48f5..3009c3a764 100644 --- a/source/lib/allocators/overrun_protector.h +++ b/source/lib/allocators/overrun_protector.h @@ -24,7 +24,7 @@ #define INCLUDED_ALLOCATORS_OVERRUN_PROTECTOR #include "lib/config2.h" // CONFIG2_ALLOCATORS_OVERRUN_PROTECTION -#include "lib/allocators/page_aligned.h" +#include "lib/sysdep/vm.h" /** OverrunProtector wraps an arbitrary object in isolated page(s) and @@ -54,7 +54,7 @@ template class OverrunProtector NONCOPYABLE(OverrunProtector); // const member public: OverrunProtector() - : object(new(page_aligned_alloc(sizeof(T))) T()) + : object(new(vm::Allocate(sizeof(T))) T()) { lock(); } @@ -63,7 +63,7 @@ public: { unlock(); object->~T(); // call dtor (since we used placement new) - page_aligned_free(object, sizeof(T)); + vm::Free(object, sizeof(T)); } T* get() const @@ -75,7 +75,7 @@ public: void lock() const { #if CONFIG2_ALLOCATORS_OVERRUN_PROTECTION - mprotect(object, sizeof(T), PROT_NONE); + vm::Protect(object, sizeof(T), PROT_NONE); #endif } @@ -83,7 +83,7 @@ private: void unlock() const { #if CONFIG2_ALLOCATORS_OVERRUN_PROTECTION - mprotect(object, sizeof(T), PROT_READ|PROT_WRITE); + vm::Protect(object, sizeof(T), PROT_READ|PROT_WRITE); #endif } diff --git a/source/lib/allocators/pool.cpp b/source/lib/allocators/pool.cpp index c47f31e40d..330c50b92e 100644 --- a/source/lib/allocators/pool.cpp +++ b/source/lib/allocators/pool.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -29,11 +29,59 @@ #include "lib/alignment.h" #include "lib/allocators/freelist.h" +#include "lib/allocators/allocator_adapters.h" #include "lib/timer.h" -TIMER_ADD_CLIENT(tc_pool_alloc); +namespace Allocators { +template +struct BasicPoolTest +{ + void operator()() const + { + Pool p(100); + const size_t initialSpace = p.RemainingObjects(); + double* p1 = p.Allocate(); + ENSURE(p1 != 0); + ENSURE(p.Contains(uintptr_t(p1))); + ENSURE(p.RemainingObjects() == initialSpace-1); + ENSURE(p.Contains(uintptr_t(p1)+1)); + ENSURE(p.Contains(uintptr_t(p1)+sizeof(double)-1)); + ENSURE(!p.Contains(uintptr_t(p1)-1)); + ENSURE(!p.Contains(uintptr_t(p1)+sizeof(double))); + if(p.RemainingObjects() == 0) + ENSURE(p.Allocate() == 0); // full + else + ENSURE(p.Allocate() != 0); // can still expand + p.DeallocateAll(); + ENSURE(!p.Contains(uintptr_t(p1))); + + p1 = p.Allocate(); + ENSURE(p1 != 0); + ENSURE(p.Contains(uintptr_t(p1))); + ENSURE(p.RemainingObjects() == initialSpace-1); + double* p2 = p.Allocate(); + ENSURE(p2 != 0); + ENSURE(p.Contains(uintptr_t(p2))); + ENSURE(p.RemainingObjects() == initialSpace-2); + ENSURE(p2 == (double*)(uintptr_t(p1)+sizeof(double))); + if(p.RemainingObjects() == 0) + ENSURE(p.Allocate() == 0); // full + else + ENSURE(p.Allocate() != 0); // can still expand + } +}; + +void TestPool() +{ + ForEachStorage(); +} + +} // namespace Allocators + + +TIMER_ADD_CLIENT(tc_pool_alloc); Status pool_create(Pool* p, size_t max_size, size_t el_size) { diff --git a/source/lib/allocators/pool.h b/source/lib/allocators/pool.h index 68aaffc1ec..3222609079 100644 --- a/source/lib/allocators/pool.h +++ b/source/lib/allocators/pool.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -21,12 +21,84 @@ */ /* - * pool allocator + * pool allocator (fixed-size blocks, freelist). */ #ifndef INCLUDED_ALLOCATORS_POOL #define INCLUDED_ALLOCATORS_POOL +#include "lib/bits.h" // ROUND_UP +#include "lib/allocators/allocator_policies.h" + +namespace Allocators { + +/** + * allocator design parameters: + * - O(1) allocation and deallocation; + * - fixed-size objects; + * - support for deallocating all objects; + * - consecutive allocations are back-to-back; + * - objects are aligned to the pointer size. + **/ +template > +class Pool +{ +public: + // (must round up because freelist stores pointers inside objects) + static const size_t objectSize = ROUND_UP(sizeof(T), sizeof(intptr_t)); + + Pool(size_t maxObjects) + : storage(maxObjects*objectSize) + { + DeallocateAll(); + } + + size_t RemainingObjects() + { + return (storage.MaxCapacity() - end) / objectSize; + } + + T* Allocate() + { + void* p = mem_freelist_Detach(freelist); + if(p) + { + ASSERT(Contains(p)); + return (T*)p; + } + + return (T*)StorageAppend(storage, end, objectSize); + } + + void Deallocate(T* p) + { + ASSERT(Contains(p)); + mem_freelist_AddToFront(freelist, p); + } + + void DeallocateAll() + { + freelist = mem_freelist_Sentinel(); + end = 0; + } + + // @return whether the address lies within the previously allocated range. + bool Contains(uintptr_t address) const + { + return (address - storage.Address()) < end; + } + +private: + Storage storage; + size_t end; + void* freelist; +}; + +LIB_API void TestPool(); + +} // namespace Allocators + + #include "lib/allocators/dynarray.h" /** @@ -145,43 +217,6 @@ LIB_API void pool_free_all(Pool* p); LIB_API size_t pool_committed(Pool* p); -/** - * C++ wrapper on top of pool_alloc for fixed-size allocations (determined by sizeof(T)) - * - * T must be POD (Plain Old Data) because it is memset to 0! - **/ -template -class PoolAllocator -{ -public: - explicit PoolAllocator(size_t maxElements) - { - (void)pool_create(&m_pool, maxElements*sizeof(T), sizeof(T)); - } - - ~PoolAllocator() - { - (void)pool_destroy(&m_pool); - } - - T* AllocateZeroedMemory() - { - T* t = (T*)pool_alloc(&m_pool, 0); - if(!t) - throw std::bad_alloc(); - memset(t, 0, sizeof(T)); - return t; - } - - void Free(T* t) - { - pool_free(&m_pool, t); - } - -private: - Pool m_pool; -}; - /** * C++ wrapper on top of pool_alloc for variable-sized allocations. * Memory is returned uninitialised. diff --git a/source/lib/allocators/tests/test_allocators.h b/source/lib/allocators/tests/test_allocators.h index 1ec7dbd0c3..c13ab0fc47 100644 --- a/source/lib/allocators/tests/test_allocators.h +++ b/source/lib/allocators/tests/test_allocators.h @@ -35,7 +35,6 @@ public: // basic test of functionality (not really meaningful) TS_ASSERT_OK(da_alloc(&da, 1000)); TS_ASSERT_OK(da_set_size(&da, 1000)); - TS_ASSERT_OK(da_set_prot(&da, PROT_NONE)); TS_ASSERT_OK(da_free(&da)); } }; diff --git a/source/lib/allocators/unique_range.cpp b/source/lib/allocators/unique_range.cpp index aeb2418d8c..1664c59a2b 100644 --- a/source/lib/allocators/unique_range.cpp +++ b/source/lib/allocators/unique_range.cpp @@ -28,6 +28,8 @@ static UniqueRangeDeleter deleters[allocationAlignment] = { FreeNone, FreeAligne static IdxDeleter numDeleters = 2; +// NB: callers should skip this if *idxDeleterOut != 0 (avoids the overhead +// of an unnecessary indirect function call) void RegisterUniqueRangeDeleter(UniqueRangeDeleter deleter, volatile IdxDeleter* idxDeleterOut) { ENSURE(deleter); @@ -44,7 +46,7 @@ void RegisterUniqueRangeDeleter(UniqueRangeDeleter deleter, volatile IdxDeleter* ENSURE(idxDeleter < (IdxDeleter)ARRAY_SIZE(deleters)); deleters[idxDeleter] = deleter; COMPILER_FENCE; - *idxDeleterOut = idxDeleter; // linearization point + *idxDeleterOut = idxDeleter; } @@ -66,8 +68,20 @@ UniqueRange AllocateAligned(size_t size, size_t alignment) const UniqueRange::pointer p = rtl_AllocateAligned(alignedSize, alignment); static volatile IdxDeleter idxDeleterAligned; - if(idxDeleterAligned == 0) + if(idxDeleterAligned == 0) // (optional optimization) RegisterUniqueRangeDeleter(FreeAligned, &idxDeleterAligned); return RVALUE(UniqueRange(p, size, idxDeleterAligned)); } + + +UniqueRange AllocateVM(size_t size, vm::PageType pageType, int prot) +{ + const UniqueRange::pointer p = vm::Allocate(size, pageType, prot); + + static volatile IdxDeleter idxDeleter; + if(idxDeleter == 0) // (optional optimization) + RegisterUniqueRangeDeleter(vm::Free, &idxDeleter); + + return RVALUE(UniqueRange(p, size, idxDeleter)); +} diff --git a/source/lib/allocators/unique_range.h b/source/lib/allocators/unique_range.h index ee051521d2..b12b619ccd 100644 --- a/source/lib/allocators/unique_range.h +++ b/source/lib/allocators/unique_range.h @@ -3,6 +3,7 @@ #include "lib/lib_api.h" #include "lib/alignment.h" // allocationAlignment +#include "lib/sysdep/vm.h" // we usually don't hold multiple references to allocations, so unique_ptr // can be used instead of the more complex (ICC generated incorrect code on @@ -191,4 +192,7 @@ static inline void swap(UniqueRange& p1, RVALUE_REF(UniqueRange) p2) LIB_API UniqueRange AllocateAligned(size_t size, size_t alignment); +LIB_API UniqueRange AllocateVM(size_t size, vm::PageType pageSize = vm::kDefault, int prot = PROT_READ|PROT_WRITE); + + #endif // #ifndef INCLUDED_ALLOCATORS_UNIQUE_RANGE diff --git a/source/lib/bits.h b/source/lib/bits.h index 71f060b213..be203cc8d9 100644 --- a/source/lib/bits.h +++ b/source/lib/bits.h @@ -229,6 +229,10 @@ inline T round_down(T n, T multiple) return result; } +// evaluates to an expression suitable as an initializer +// for constant static data members. +#define ROUND_UP(n, multiple) (((n) + (multiple)-1) & ~((multiple)-1)) + template inline T MaxPowerOfTwoDivisor(T value) diff --git a/source/lib/debug.cpp b/source/lib/debug.cpp index 1a96843a9d..46d1986bdb 100644 --- a/source/lib/debug.cpp +++ b/source/lib/debug.cpp @@ -33,8 +33,8 @@ #include "lib/alignment.h" #include "lib/app_hooks.h" -#include "lib/allocators/page_aligned.h" #include "lib/fnv_hash.h" +#include "lib/sysdep/vm.h" #include "lib/sysdep/cpu.h" // cpu_CAS #include "lib/sysdep/sysdep.h" @@ -207,7 +207,7 @@ static const size_t messageSize = 512*KiB; void debug_FreeErrorMessage(ErrorMessageMem* emm) { - page_aligned_free(emm->pa_mem, messageSize); + vm::Free(emm->pa_mem, messageSize); } @@ -274,7 +274,7 @@ const wchar_t* debug_BuildErrorMessage( sys_StatusDescription(0, os_error, ARRAY_SIZE(os_error)); // rationale: see ErrorMessageMem - emm->pa_mem = page_aligned_alloc(messageSize); + emm->pa_mem = vm::Allocate(messageSize); wchar_t* const buf = (wchar_t*)emm->pa_mem; if(!buf) return L"(insufficient memory to generate error message)"; diff --git a/source/lib/debug.h b/source/lib/debug.h index f6525a0b46..aad2a1b171 100644 --- a/source/lib/debug.h +++ b/source/lib/debug.h @@ -537,8 +537,8 @@ struct ErrorMessageMem // rationale: // - error messages with stack traces require a good deal of memory // (hundreds of KB). static buffers of that size are undesirable. - // - the heap may be corrupted, so don't use malloc. allocator.h's - // page_aligned_malloc (implemented via mmap) should be safe. + // - the heap may be corrupted, so don't use malloc. + // instead, "lib/sysdep/vm.h" functions should be safe. // - alloca is a bit iffy (the stack may be maxed out), non-portable and // complicates the code because it can't be allocated by a subroutine. // - this method is probably slow, but error messages aren't built often. diff --git a/source/lib/pch/pch_warnings.h b/source/lib/pch/pch_warnings.h index 95b1c45cf6..e6e65cb4eb 100644 --- a/source/lib/pch/pch_warnings.h +++ b/source/lib/pch/pch_warnings.h @@ -10,6 +10,7 @@ // .. always disabled W4 # pragma warning(disable:4103) // alignment changed after including header (boost has #pragma pack/pop in separate headers) # pragma warning(disable:4127) // conditional expression is constant; rationale: see STMT in lib.h. +# pragma warning(disable:4324) // structure was padded due to __declspec(align()) # pragma warning(disable:4351) // yes, default init of array entries is desired # pragma warning(disable:4355) // 'this' used in base member initializer list # pragma warning(disable:4718) // recursive call has no side effects, deleting diff --git a/source/lib/sysdep/os/unix/uvm.cpp b/source/lib/sysdep/os/unix/uvm.cpp new file mode 100644 index 0000000000..8112eb1e67 --- /dev/null +++ b/source/lib/sysdep/os/unix/uvm.cpp @@ -0,0 +1,136 @@ +/* Copyright (c) 2011 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "precompiled.h" +#include "lib/sysdep/vm.h" + +#include "lib/alignment.h" + +// "anonymous" effectively means mapping /dev/zero, but is more efficient. +// MAP_ANONYMOUS is not in SUSv3, but is a very common extension. +// unfortunately, MacOS X only defines MAP_ANON, which Solaris says is +// deprecated. workaround there: define MAP_ANONYMOUS in terms of MAP_ANON. +#ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +#endif + +static const int mmap_flags = MAP_PRIVATE|MAP_ANONYMOUS; + +namespace vm { + +void* ReserveAddressSpace(size_t size, size_t UNUSED(commitSize), PageType UNUSED(pageType), int UNUSED(prot)) +{ + errno = 0; + void* p = mmap(0, size, PROT_NONE, mmap_flags|MAP_NORESERVE, -1, 0); + if(p == MAP_FAILED) + return 0; + return p; +} + +void ReleaseAddressSpace(void* p, size_t size) +{ + ENSURE(size != 0); + + errno = 0; + if(munmap(p, size) != 0) + DEBUG_WARN_ERR(StatusFromErrno()); +} + + +bool Commit(uintptr_t address, size_t size, PageType UNUSED(pageType), int prot) +{ + if(prot == PROT_NONE) // would be understood as a request to decommit + { + DEBUG_WARN_ERR(ERR::INVALID_PARAM); + return false; + } + + errno = 0; + if(mmap((void*)address, size, prot, mmap_flags|MAP_FIXED, -1, 0) == MAP_FAILED) + return false; + + if(prot != (PROT_READ|PROT_WRITE)) + (void)Protect(address, size, prot); + + return true; +} + +bool Decommit(void* p, size_t size) +{ + errno = 0; + if(mmap(p, size, PROT_NONE, mmap_flags|MAP_NORESERVE|MAP_FIXED, -1, 0) == MAP_FAILED) + return false; + return true; +} + + +bool Protect(uintptr_t address, size_t size, int prot) +{ + errno = 0; + if(mprotect((void*)address, size, prot) != 0) + { + DEBUG_WARN_ERR(ERR::FAIL); + return false; + } + return true; +} + + +void* Allocate(size_t size, PageType pageType, int prot) +{ + void* p = ReserveAddressSpace(size); + if(!p) + return 0; + + if(!Commit(uintptr_t(p), size, pageType, prot)) + { + ReleaseAddressSpace(p, size); + return 0; + } + + return p; +} + +void Free(void* p, size_t size) +{ + // (only the Windows implementation distinguishes between Free and ReleaseAddressSpace) + vm::ReleaseAddressSpace(p, size); +} + + +void BeginOnDemandCommits() +{ + // not yet implemented, but possible with a signal handler +} + +void EndOnDemandCommits() +{ + // not yet implemented, but possible with a signal handler +} + + +void DumpStatistics() +{ + // we haven't collected any statistics +} + +} // namespace vm diff --git a/source/lib/sysdep/os/win/wnuma.cpp b/source/lib/sysdep/os/win/wnuma.cpp index 179f0e597b..ae57b26dde 100644 --- a/source/lib/sysdep/os/win/wnuma.cpp +++ b/source/lib/sysdep/os/win/wnuma.cpp @@ -27,9 +27,9 @@ #include "lib/alignment.h" #include "lib/timer.h" #include "lib/module_init.h" -#include "lib/allocators/page_aligned.h" -#include "lib/sysdep/os_cpu.h" +#include "lib/sysdep/vm.h" #include "lib/sysdep/acpi.h" +#include "lib/sysdep/os_cpu.h" #include "lib/sysdep/os/win/win.h" #include "lib/sysdep/os/win/wutil.h" #include "lib/sysdep/os/win/wcpu.h" @@ -374,7 +374,7 @@ static double ReadRelativeDistanceFromSLIT(const SLIT* slit) static double MeasureRelativeDistance() { const size_t size = 32*MiB; - void* mem = page_aligned_alloc(size); + void* mem = vm::Allocate(size); ASSUME_ALIGNED(mem, pageSize); const uintptr_t previousProcessorMask = os_cpu_SetThreadAffinityMask(os_cpu_ProcessorMask()); @@ -395,7 +395,7 @@ static double MeasureRelativeDistance() (void)os_cpu_SetThreadAffinityMask(previousProcessorMask); - page_aligned_free(mem, size); + vm::Free(mem, size); return maxTime / minTime; } @@ -462,81 +462,54 @@ bool numa_IsMemoryInterleaved() //----------------------------------------------------------------------------- -// allocator -// -//static bool VerifyPages(void* mem, size_t size, size_t pageSize, size_t node) -//{ -// WUTIL_FUNC(pQueryWorkingSetEx, BOOL, (HANDLE, PVOID, DWORD)); -// WUTIL_IMPORT_KERNEL32(QueryWorkingSetEx, pQueryWorkingSetEx); -// if(!pQueryWorkingSetEx) -// return true; // can't do anything -// -//#if WINVER >= 0x600 -// size_t largePageSize = os_cpu_LargePageSize(); -// ENSURE(largePageSize != 0); // this value is needed for later -// -// // retrieve attributes of all pages constituting mem -// const size_t numPages = (size + pageSize-1) / pageSize; -// PSAPI_WORKING_SET_EX_INFORMATION* wsi = new PSAPI_WORKING_SET_EX_INFORMATION[numPages]; -// for(size_t i = 0; i < numPages; i++) -// wsi[i].VirtualAddress = (u8*)mem + i*pageSize; -// pQueryWorkingSetEx(GetCurrentProcess(), wsi, DWORD(sizeof(PSAPI_WORKING_SET_EX_INFORMATION)*numPages)); -// -// // ensure each is valid and allocated on the correct node -// for(size_t i = 0; i < numPages; i++) -// { -// const PSAPI_WORKING_SET_EX_BLOCK& attributes = wsi[i].VirtualAttributes; -// if(!attributes.Valid) -// return false; -// if((attributes.LargePage != 0) != (pageSize == largePageSize)) -// { -// debug_printf(L"NUMA: is not a large page\n"); -// return false; -// } -// if(attributes.Node != node) -// { -// debug_printf(L"NUMA: allocated from remote node\n"); -// return false; -// } -// } -// -// delete[] wsi; -//#else -// UNUSED2(mem); -// UNUSED2(size); -// UNUSED2(pageSize); -// UNUSED2(node); -//#endif -// -// return true; -//} -// -// -//void* numa_AllocateOnNode(size_t node, size_t size, LargePageDisposition largePageDisposition, size_t* ppageSize) -//{ -// ENSURE(node < numa_NumNodes()); -// -// // see if there will be enough memory (non-authoritative, for debug purposes only) -// { -// const size_t sizeMiB = size/MiB; -// const size_t availableMiB = numa_AvailableMemory(node); -// if(availableMiB < sizeMiB) -// debug_printf(L"NUMA: warning: node reports insufficient memory (%d vs %d MB)\n", availableMiB, sizeMiB); -// } -// -// size_t pageSize; // (used below even if ppageSize is zero) -// void* const mem = numa_Allocate(size, largePageDisposition, &pageSize); -// if(ppageSize) -// *ppageSize = pageSize; -// -// // we can't use VirtualAllocExNuma - it's only available in Vista and Server 2008. -// // workaround: fault in all pages now to ensure they are allocated from the -// // current node, then verify page attributes. -// const uintptr_t previousProcessorMask = os_cpu_SetThreadAffinityMask(numa_ProcessorMaskFromNode(node)); -// memset(mem, 0, size); -// (void)os_cpu_SetThreadAffinityMask(previousProcessorMask); -// -// VerifyPages(mem, size, pageSize, node); -// -// return mem; -//} + +#if 0 + +static bool VerifyPages(void* mem, size_t size, size_t pageSize, size_t node) +{ + WUTIL_FUNC(pQueryWorkingSetEx, BOOL, (HANDLE, PVOID, DWORD)); + WUTIL_IMPORT_KERNEL32(QueryWorkingSetEx, pQueryWorkingSetEx); + if(!pQueryWorkingSetEx) + return true; // can't do anything + +#if WINVER >= 0x600 + size_t largePageSize = os_cpu_LargePageSize(); + ENSURE(largePageSize != 0); // this value is needed for later + + // retrieve attributes of all pages constituting mem + const size_t numPages = (size + pageSize-1) / pageSize; + PSAPI_WORKING_SET_EX_INFORMATION* wsi = new PSAPI_WORKING_SET_EX_INFORMATION[numPages]; + for(size_t i = 0; i < numPages; i++) + wsi[i].VirtualAddress = (u8*)mem + i*pageSize; + pQueryWorkingSetEx(GetCurrentProcess(), wsi, DWORD(sizeof(PSAPI_WORKING_SET_EX_INFORMATION)*numPages)); + + // ensure each is valid and allocated on the correct node + for(size_t i = 0; i < numPages; i++) + { + const PSAPI_WORKING_SET_EX_BLOCK& attributes = wsi[i].VirtualAttributes; + if(!attributes.Valid) + return false; + if((attributes.LargePage != 0) != (pageSize == largePageSize)) + { + debug_printf(L"NUMA: is not a large page\n"); + return false; + } + if(attributes.Node != node) + { + debug_printf(L"NUMA: allocated from remote node\n"); + return false; + } + } + + delete[] wsi; +#else + UNUSED2(mem); + UNUSED2(size); + UNUSED2(pageSize); + UNUSED2(node); +#endif + + return true; +} + +#endif diff --git a/source/lib/sysdep/os/win/wposix/wmman.cpp b/source/lib/sysdep/os/win/wposix/wmman.cpp index e058a548c0..bf32e02d83 100644 --- a/source/lib/sysdep/os/win/wposix/wmman.cpp +++ b/source/lib/sysdep/os/win/wposix/wmman.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -27,20 +27,14 @@ #include "lib/sysdep/os/win/wposix/crt_posix.h" // _get_osfhandle -//----------------------------------------------------------------------------- -// memory mapping -//----------------------------------------------------------------------------- - -// convert POSIX PROT_* flags to their Win32 PAGE_* enumeration equivalents. -// used by mprotect. -static DWORD win32_prot(int prot) +unsigned MemoryProtectionFromPosix(int prot) { + if(prot == PROT_NONE) + return PAGE_NOACCESS; + // this covers all 8 combinations of read|write|exec - // (note that "none" means all flags are 0). switch(prot & (PROT_READ|PROT_WRITE|PROT_EXEC)) { - case PROT_NONE: - return PAGE_NOACCESS; case PROT_READ: return PAGE_READONLY; case PROT_WRITE: @@ -57,15 +51,22 @@ static DWORD win32_prot(int prot) return PAGE_EXECUTE_READWRITE; case PROT_READ|PROT_WRITE|PROT_EXEC: return PAGE_EXECUTE_READWRITE; + default: // none set + DEBUG_WARN_ERR(ERR::INVALID_FLAG); + return PAGE_NOACCESS; } - return 0; // UNREACHABLE + // UNREACHABLE } +//----------------------------------------------------------------------------- +// memory mapping +//----------------------------------------------------------------------------- + int mprotect(void* addr, size_t len, int prot) { - const DWORD newProtect = win32_prot(prot); + const DWORD newProtect = (DWORD)MemoryProtectionFromPosix(prot); DWORD oldProtect; // required by VirtualProtect const BOOL ok = VirtualProtect(addr, len, newProtect, &oldProtect); WARN_IF_FALSE(ok); @@ -104,7 +105,7 @@ static Status mmap_mem(void* start, size_t len, int prot, int flags, int fd, voi } const DWORD allocationType = want_commit? MEM_COMMIT : MEM_RESERVE; - const DWORD protect = win32_prot(prot); + const DWORD protect = (DWORD)MemoryProtectionFromPosix(prot); void* p = VirtualAlloc(start, len, allocationType, protect); if(!p) { diff --git a/source/lib/sysdep/os/win/wposix/wmman.h b/source/lib/sysdep/os/win/wposix/wmman.h index fcbbbe093e..810b53873e 100644 --- a/source/lib/sysdep/os/win/wposix/wmman.h +++ b/source/lib/sysdep/os/win/wposix/wmman.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -56,4 +56,7 @@ extern int munmap(void* start, size_t len); extern int mprotect(void* addr, size_t len, int prot); +// convert POSIX PROT_* flags to their Win32 PAGE_* enumeration equivalents. +LIB_API unsigned MemoryProtectionFromPosix(int prot); + #endif // #ifndef INCLUDED_WMMAN diff --git a/source/lib/sysdep/os/win/wvm.cpp b/source/lib/sysdep/os/win/wvm.cpp new file mode 100644 index 0000000000..ddbe10ca03 --- /dev/null +++ b/source/lib/sysdep/os/win/wvm.cpp @@ -0,0 +1,539 @@ +/* Copyright (c) 2011 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * virtual memory interface. supercedes POSIX mmap; provides support for + * large pages, autocommit, and specifying protection flags during allocation. + */ + +#include "precompiled.h" +#include "lib/sysdep/vm.h" + +#include "lib/sysdep/os/win/wutil.h" +#include + +#include "lib/timer.h" +#include "lib/bits.h" // round_down +#include "lib/alignment.h" // CACHE_ALIGNED +#include "lib/module_init.h" +#include "lib/sysdep/cpu.h" // cpu_AtomicAdd +#include "lib/sysdep/numa.h" +#include "lib/sysdep/arch/x86_x64/x86_x64.h" // x86_x64_ApicId +#include "lib/sysdep/arch/x86_x64/topology.h" // cpu_topology_ProcessorFromApicId +#include "lib/sysdep/os/win/wversion.h" +#include "lib/sysdep/os/win/winit.h" +WINIT_REGISTER_CRITICAL_INIT(wvm_Init); + + +//----------------------------------------------------------------------------- +// functions not supported by 32-bit Windows XP + +static WUTIL_FUNC(pGetCurrentProcessorNumber, DWORD, (VOID)); +static WUTIL_FUNC(pGetNumaProcessorNode, BOOL, (UCHAR, PUCHAR)); +static WUTIL_FUNC(pVirtualAllocExNuma, LPVOID, (HANDLE, LPVOID, SIZE_T, DWORD, DWORD, DWORD)); + +static DWORD WINAPI EmulateGetCurrentProcessorNumber(VOID) +{ + const u8 apicId = x86_x64_ApicId(); + const DWORD processor = cpu_topology_ProcessorFromApicId(apicId); + ASSERT(processor < os_cpu_MaxProcessors); + return processor; +} + +static BOOL WINAPI EmulateGetNumaProcessorNode(UCHAR UNUSED(processor), PUCHAR node) +{ + // given that the system doesn't support GetNumaProcessorNode, + // it will also lack VirtualAllocExNuma, so the node value we assign + // is ignored by EmulateVirtualAllocExNuma. + *node = 0; + return TRUE; +} + +static LPVOID WINAPI EmulateVirtualAllocExNuma(HANDLE UNUSED(hProcess), LPVOID p, SIZE_T size, DWORD allocationType, DWORD protect, DWORD UNUSED(node)) +{ + return VirtualAlloc(p, size, allocationType, protect); +} + + +static Status wvm_Init() +{ + WUTIL_IMPORT_KERNEL32(GetCurrentProcessorNumber, pGetCurrentProcessorNumber); + WUTIL_IMPORT_KERNEL32(GetNumaProcessorNode, pGetNumaProcessorNode); + WUTIL_IMPORT_KERNEL32(VirtualAllocExNuma, pVirtualAllocExNuma); + + if(!pGetCurrentProcessorNumber) + pGetCurrentProcessorNumber = &EmulateGetCurrentProcessorNumber; + if(!pGetNumaProcessorNode) + pGetNumaProcessorNode = &EmulateGetNumaProcessorNode; + if(!pVirtualAllocExNuma) + pVirtualAllocExNuma = &EmulateVirtualAllocExNuma; + + return INFO::OK; +} + + +namespace vm { + + +//----------------------------------------------------------------------------- +// per-processor statistics + +// (alignment avoids false sharing) +CACHE_ALIGNED(struct Statistics) // POD +{ + // thread-safe (required due to concurrent commits) + void NotifyLargePageCommit() + { + cpu_AtomicAdd(&largePageCommits, +1); + } + + void NotifySmallPageCommit() + { + cpu_AtomicAdd(&smallPageCommits, +1); + } + + intptr_t largePageCommits; + intptr_t smallPageCommits; +}; +static CACHE_ALIGNED(Statistics) statistics[os_cpu_MaxProcessors]; + +void DumpStatistics() +{ + ENSURE(IsAligned(&statistics[0], cacheLineSize)); + ENSURE(IsAligned(&statistics[1], cacheLineSize)); + + size_t smallPageCommits = 0; + size_t largePageCommits = 0; + uintptr_t processorsWithNoCommits = 0; + for(size_t processor = 0; processor < os_cpu_NumProcessors(); processor++) + { + const Statistics& s = statistics[processor]; + if(s.smallPageCommits == 0 && s.largePageCommits == 0) + processorsWithNoCommits |= Bit(processor); + smallPageCommits += s.smallPageCommits; + largePageCommits += s.largePageCommits; + } + + const size_t totalCommits = smallPageCommits+largePageCommits; + if(totalCommits == 0) // this module wasn't used => don't print debug output + return; + + const size_t largePageRatio = totalCommits? largePageCommits*100/totalCommits : 0; + debug_printf(L"%d commits (%d, i.e. %d%% of them via large pages)\n", totalCommits, largePageCommits, largePageRatio); + if(processorsWithNoCommits != 0) + debug_printf(L" processors with no commits: %x\n", processorsWithNoCommits); + + if(numa_NumNodes() > 1) + debug_printf(L"NUMA factor: %.2f\n", numa_Factor()); +} + + +//----------------------------------------------------------------------------- +// allocator with large-page and NUMA support + +static bool largePageAllocationTookTooLong = false; + +static bool ShouldUseLargePages(size_t allocationSize, DWORD allocationType, PageType pageType) +{ + // don't even check for large page support. + if(pageType == kSmall) + return false; + + // can't use large pages when reserving - VirtualAlloc would fail with + // ERROR_INVALID_PARAMETER. + if((allocationType & MEM_COMMIT) == 0) + return false; + + // OS lacks support for large pages. + if(os_cpu_LargePageSize() == 0) + return false; + + // large pages are available and application wants them used. + if(pageType == kLarge) + return true; + + // default: use a heuristic. + { + // internal fragmentation would be excessive. + if(allocationSize <= largePageSize/2) + return false; + + // a previous attempt already took too long. + if(largePageAllocationTookTooLong) + return false; + + // pre-Vista Windows OSes attempt to cope with page fragmentation by + // trimming the working set of all processes, thus swapping them out, + // and waiting for contiguous regions to appear. this is terribly + // slow (multiple seconds), hence the following heuristic: + if(wversion_Number() < WVERSION_VISTA) + { + // if there's not plenty of free memory, then memory is surely + // already fragmented. + if(os_cpu_MemoryAvailable() < 2000) // 2 GB + return false; + } + } + + return true; +} + + +// used for reserving address space, committing pages, or both. +static void* AllocateLargeOrSmallPages(uintptr_t address, size_t size, DWORD allocationType, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE) +{ + const HANDLE hProcess = GetCurrentProcess(); + const DWORD protect = MemoryProtectionFromPosix(prot); + + UCHAR node; + const DWORD processor = pGetCurrentProcessorNumber(); + WARN_IF_FALSE(pGetNumaProcessorNode((UCHAR)processor, &node)); + + if(ShouldUseLargePages(size, allocationType, pageType)) + { + // MEM_LARGE_PAGES requires aligned addresses and sizes + const size_t largePageSize = os_cpu_LargePageSize(); + const uintptr_t alignedAddress = round_down(address, largePageSize); + const size_t alignedSize = round_up(size+largePageSize-1, largePageSize); + // note: this call can take SECONDS, which is why several checks are + // undertaken before we even try. these aren't authoritative, so we + // at least prevent future attempts if it takes too long. + const double startTime = timer_Time(); COMPILER_FENCE; + void* largePages = pVirtualAllocExNuma(hProcess, LPVOID(alignedAddress), alignedSize, allocationType|MEM_LARGE_PAGES, protect, node); + const double elapsedTime = timer_Time() - startTime; COMPILER_FENCE; + if(elapsedTime > 0.5) + largePageAllocationTookTooLong = true; // avoid large pages next time + if(largePages) + { + if((allocationType & MEM_COMMIT) != 0) + statistics[processor].NotifyLargePageCommit(); + return largePages; + } + } + + // try (again) with regular pages + void* smallPages = pVirtualAllocExNuma(hProcess, LPVOID(address), size, allocationType, protect, node); + if(smallPages) + { + if((allocationType & MEM_COMMIT) != 0) + statistics[processor].NotifySmallPageCommit(); + return smallPages; + } + + return 0; +} + + +//----------------------------------------------------------------------------- +// address space reservation + +// indicates the extent of a range of address space, +// and the parameters for committing large/small pages in it. +// +// this bookkeeping information increases the safety of on-demand commits, +// enables different parameters for separate allocations, and allows +// variable alignment because it retains the original base address. +// (storing this information within the allocated memory would +// require mapping an additional page and may waste an entire +// large page if the base address happens to be aligned already.) +CACHE_ALIGNED(struct AddressRangeDescriptor) // POD +{ + // attempt to activate this descriptor and reserve address space. + // side effect: initializes all fields if successful. + // + // @param size, commitSize, pageType, prot - see ReserveAddressSpace. + // @return INFO::SKIPPED if this descriptor is already in use, + // INFO::OK on success, otherwise ERR::NO_MEM (after showing an + // error message). + Status Allocate(size_t size, size_t commitSize, PageType pageType, int prot) + { + // if this descriptor wasn't yet in use, mark it as busy + // (double-checking is cheaper than cpu_CAS) + if(base != 0 || !cpu_CAS(&base, intptr_t(0), intptr_t(this))) + return INFO::SKIPPED; + + ENSURE(size != 0); // probably indicates a bug in caller + ENSURE((commitSize % largePageSize) == 0 || pageType == kSmall); + ASSERT(pageType == kLarge || pageType == kSmall || pageType == kDefault); + ASSERT(prot == PROT_NONE || (prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC)) == 0); + this->commitSize = commitSize; + this->pageType = pageType; + this->prot = prot; + + alignment = (pageType == kSmall)? pageSize : largePageSize; + totalSize = round_up(size+alignment-1, alignment); + + // NB: it is meaningless to ask for large pages when reserving + // (see ShouldUseLargePages). pageType only affects subsequent commits. + base = (intptr_t)AllocateLargeOrSmallPages(0, totalSize, MEM_RESERVE); + if(!base) + { + DEBUG_DISPLAY_ERROR(ErrorString()); + return ERR::NO_MEM; // NOWARN (error string is more helpful) + } + + alignedBase = round_up(uintptr_t(base), alignment); + alignedEnd = alignedBase + round_up(size, alignment); + return INFO::OK; + } + + void Free() + { + vm::Free((void*)base, totalSize); + alignment = alignedBase = alignedEnd = 0; + totalSize = 0; + COMPILER_FENCE; + base = 0; // release descriptor for subsequent reuse + } + + bool Contains(uintptr_t address) const + { + // safety check: we should never see pointers in the no-man's-land + // between the original and rounded up base addresses. + ENSURE(!(uintptr_t(base) <= address && address < alignedBase)); + + return (alignedBase <= address && address < alignedEnd); + } + + bool Commit(uintptr_t address) + { + // (safe because Allocate rounded up to alignment) + const uintptr_t alignedAddress = round_down(address, alignment); + ENSURE(alignedBase <= alignedAddress && alignedAddress+commitSize <= alignedEnd); + return vm::Commit(alignedAddress, commitSize, pageType, prot); + } + + // corresponds to the respective page size (Windows requires + // naturally aligned addresses and sizes when committing large pages). + // note that VirtualAlloc's alignment defaults to 64 KiB. + uintptr_t alignment; + + uintptr_t alignedBase; // multiple of alignment + uintptr_t alignedEnd; // " + + // (actual requested size / allocated address is required by + // ReleaseAddressSpace due to variable alignment.) + volatile intptr_t base; // (type is dictated by cpu_CAS) + size_t totalSize; + + // parameters to be relayed to vm::Commit + size_t commitSize; + PageType pageType; + int prot; + +//private: + static const wchar_t* ErrorString() + { +#if ARCH_IA32 + return L"Out of address space (64-bit OS is required)"; +#elif OS_WIN + // because early AMD64 lacked CMPXCHG16B, the Windows lock-free slist + // must squeeze the address, ABA tag and list length (a questionable + // design decision) into 64 bits. that leaves 39 bits for the + // address, plus 4 implied zero bits due to 16-byte alignment. + // [http://www.alex-ionescu.com/?p=50] + return L"Out of address space (Windows only provides 8 TiB)"; +#else + return L"Out of address space"; +#endif + } +}; + +// (array size governs the max. number of extant allocations) +static AddressRangeDescriptor ranges[2*os_cpu_MaxProcessors]; + + +static AddressRangeDescriptor* FindDescriptor(uintptr_t address) +{ + for(size_t idxRange = 0; idxRange < ARRAY_SIZE(ranges); idxRange++) + { + AddressRangeDescriptor& d = ranges[idxRange]; + if(d.Contains(address)) + return &d; + } + + return 0; // not contained in any allocated ranges +} + + +void* ReserveAddressSpace(size_t size, size_t commitSize, PageType pageType, int prot) +{ + for(size_t idxRange = 0; idxRange < ARRAY_SIZE(ranges); idxRange++) + { + Status ret = ranges[idxRange].Allocate(size, commitSize, pageType, prot); + if(ret == INFO::OK) + return (void*)ranges[idxRange].alignedBase; + if(ret == ERR::NO_MEM) + return 0; + // else: descriptor already in use, try the next one + } + + // all descriptors are in use; ranges[] was too small + DEBUG_WARN_ERR(ERR::LIMIT); + return 0; +} + + +void ReleaseAddressSpace(void* p, size_t UNUSED(size)) +{ + // it is customary to ignore null pointers + if(!p) + return; + + AddressRangeDescriptor* d = FindDescriptor(uintptr_t(p)); + if(d) + d->Free(); + else + { + debug_printf(L"No AddressRangeDescriptor contains %P\n", p); + ENSURE(0); + } +} + + +//----------------------------------------------------------------------------- +// commit/decommit, allocate/free, protect + +// [23 page faults for an 8 MPixel image total 789 kc, i.e. < 1 ms] +TIMER_ADD_CLIENT(tc_commit); + +bool Commit(uintptr_t address, size_t size, PageType pageType, int prot) +{ + TIMER_ACCRUE_ATOMIC(tc_commit); + + return AllocateLargeOrSmallPages(address, size, MEM_COMMIT, pageType, prot) != 0; +} + + +bool Decommit(uintptr_t address, size_t size) +{ + return VirtualFree(LPVOID(address), size, MEM_DECOMMIT) != FALSE; +} + + +bool Protect(uintptr_t address, size_t size, int prot) +{ + const DWORD protect = MemoryProtectionFromPosix(prot); + DWORD oldProtect; // required by VirtualProtect + const BOOL ok = VirtualProtect(LPVOID(address), size, protect, &oldProtect); + return ok != FALSE; +} + + +void* Allocate(size_t size, PageType pageType, int prot) +{ + return AllocateLargeOrSmallPages(0, size, MEM_RESERVE|MEM_COMMIT, pageType, prot); +} + + +void Free(void* p, size_t UNUSED(size)) +{ + if(p) // otherwise, VirtualFree complains + { + const BOOL ok = VirtualFree(p, 0, MEM_RELEASE); + WARN_IF_FALSE(ok); + } +} + + +//----------------------------------------------------------------------------- +// on-demand commit + +static LONG CALLBACK VectoredHandler(const PEXCEPTION_POINTERS ep) +{ + const PEXCEPTION_RECORD er = ep->ExceptionRecord; + + // we only want to handle access violations. (strictly speaking, + // unmapped memory causes page faults, but Windows reports them + // with EXCEPTION_ACCESS_VIOLATION.) + if(er->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) + return EXCEPTION_CONTINUE_SEARCH; + + // NB: read exceptions are legitimate and occur when updating an + // accumulator for the first time. + + // get the source/destination of the read/write operation that + // failed. (NB: don't use er->ExceptionAddress - that's the + // location of the code that encountered the fault) + const uintptr_t address = (uintptr_t)er->ExceptionInformation[1]; + + // if unknown (e.g. access violation in kernel address space or + // violation of alignment requirements), we don't want to handle it. + if(address == ~uintptr_t(0)) + return EXCEPTION_CONTINUE_SEARCH; + + // the address space must have been allocated by ReserveAddressSpace + // (otherwise we wouldn't know the desired commitSize/pageType/prot). + AddressRangeDescriptor* d = FindDescriptor(address); + if(!d) + return EXCEPTION_CONTINUE_SEARCH; + + // NB: the first access to a page isn't necessarily at offset 0 + // (memcpy isn't guaranteed to copy sequentially). rounding down + // is safe and necessary - see AddressRangeDescriptor::alignment. + const uintptr_t alignedAddress = round_down(address, d->alignment); + bool ok = d->Commit(alignedAddress); + if(!ok) + { + debug_printf(L"VectoredHandler: Commit(0x%p) failed; address=0x%p\n", alignedAddress, address); + ENSURE(0); + return EXCEPTION_CONTINUE_SEARCH; + } + + // continue at (i.e. retry) the same instruction. + return EXCEPTION_CONTINUE_EXECUTION; +} + + +static PVOID handler; +static ModuleInitState initState; +static volatile intptr_t references = 0; // atomic + +static Status InitHandler() +{ + ENSURE(handler == 0); + handler = AddVectoredExceptionHandler(TRUE, VectoredHandler); + ENSURE(handler != 0); + return INFO::OK; +} + +static void ShutdownHandler() +{ + ENSURE(handler != 0); + const ULONG ret = RemoveVectoredExceptionHandler(handler); + ENSURE(ret != 0); + handler = 0; +} + +void BeginOnDemandCommits() +{ + ModuleInit(&initState, InitHandler); + cpu_AtomicAdd(&references, +1); +} + +void EndOnDemandCommits() +{ + if(cpu_AtomicAdd(&references, -1) == 1) + ModuleShutdown(&initState, ShutdownHandler); +} + +} // namespace vm diff --git a/source/lib/sysdep/vm.h b/source/lib/sysdep/vm.h new file mode 100644 index 0000000000..a3d1068bd8 --- /dev/null +++ b/source/lib/sysdep/vm.h @@ -0,0 +1,154 @@ +/* Copyright (c) 2011 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * virtual memory interface. supercedes POSIX mmap; provides support for + * large pages, autocommit, and specifying protection flags during allocation. + */ + +#ifndef INCLUDED_SYSDEP_VM +#define INCLUDED_SYSDEP_VM + +#include "lib/posix/posix_mman.h" // PROT_* + +namespace vm { + +// committing large pages (2 MiB) instead of regular 4 KiB pages can +// increase TLB coverage and reduce misses for sequential access patterns. +// however, small page TLBs have more entries, making them better suited +// to random accesses. it may also take a long time to find/free up +// contiguous regions of physical memory for large pages. applications +// can express their preference or go along with the default, +// which depends on several factors such as allocation size. +enum PageType +{ + kLarge, // use large if available + kSmall, // always use small + kDefault // heuristic +}; + +/** + * reserve address space and set the parameters for any later + * on-demand commits. + * + * @param size desired number of bytes. any additional space + * in the last page is also accessible. + * @param commitSize [bytes] how much to commit each time. + * larger values reduce the number of page faults at the cost of + * additional internal fragmentation. must be a multiple of + * largePageSize unless pageType == kSmall. + * @param pageType chooses between large/small pages for commits. + * @param prot memory protection flags for newly committed pages. + * @return base address (aligned to the respective page size) or + * 0 if address space/descriptor storage is exhausted + * (an error dialog will also be raised). + * must be freed via ReleaseAddressSpace. +**/ +LIB_API void* ReserveAddressSpace(size_t size, size_t commitSize = largePageSize, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE); + +/** + * release address space and decommit any memory. + * + * @param p a pointer previously returned by ReserveAddressSpace. + * @param size is required by the POSIX implementation and + * ignored on Windows. it also ensures compatibility with UniqueRange. + **/ +LIB_API void ReleaseAddressSpace(void* p, size_t size = 0); + + +/** + * map physical memory to previously reserved address space. + * + * @param address, size need not be aligned, but this function commits + * any pages intersecting that interval. + * @param pageType, prot - see ReserveAddressSpace. + * @return whether memory was successfully committed. + * + * note: committing only maps virtual pages and does not actually allocate + * page frames. Windows XP uses a first-touch heuristic - the page will + * be taken from the node whose processor caused the fault. + * therefore, worker threads should be the first to write to their memory. + * + * (this is surprisingly slow in XP, possibly due to PFN lock contention) + **/ +LIB_API bool Commit(uintptr_t address, size_t size, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE); + +/** + * unmap physical memory. + * + * @return whether the operation succeeded. + **/ +LIB_API bool Decommit(uintptr_t address, size_t size); + + +/** + * set the memory protection flags for all pages that intersect + * the given interval. + * the pages must currently be committed. + * + * @param prot memory protection flags: PROT_NONE or a combination of + * PROT_READ, PROT_WRITE, PROT_EXEC. + **/ +LIB_API bool Protect(uintptr_t address, size_t size, int prot); + + +/** + * reserve address space and commit memory. + * + * @param size [bytes] to allocate. + * @param pageType, prot - see ReserveAddressSpace. + * @return zero-initialized memory aligned to the respective + * page size. + **/ +LIB_API void* Allocate(size_t size, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE); + +/** + * decommit memory and release address space. + * + * @param p a pointer previously returned by Allocate. + * @param size is required by the POSIX implementation and + * ignored on Windows. it also ensures compatibility with UniqueRange. + * + * (this differs from ReleaseAddressSpace, which must account for + * extra padding/alignment to largePageSize.) + **/ +LIB_API void Free(void* p, size_t size = 0); + + +/** + * install a handler that attempts to commit memory whenever a + * read/write page fault is encountered. thread-safe. + **/ +LIB_API void BeginOnDemandCommits(); + +/** + * decrements the reference count begun by BeginOnDemandCommit and + * removes the page fault handler when it reaches 0. thread-safe. + **/ +LIB_API void EndOnDemandCommits(); + + +LIB_API void DumpStatistics(); + +} // namespace vm + +#endif // #ifndef INCLUDED_SYSDEP_VM