lay groundwork for more efficient and flexible allocators. add new sysdep/vm that provides access to additional features on Windows (large pages, autocommit). add Pool/Arena allocators that avoid overhead and support arbitrary storage (not just the expensive virtual memory allocator in DynArray)

This was SVN commit r10051.

source/lib/alignment.h

@@ -65,7 +65,7 @@ static const size_t largePageSize = 0x200000;	// 2 MB
 // misc
 //
 
-static const size_t allocationAlignment = ARCH_AMD64? 16 : 8;
+static const size_t allocationAlignment = 16;
 
 static const size_t KiB = size_t(1) << 10;
 static const size_t MiB = size_t(1) << 20;

source/lib/allocators/allocator_adapters.h

@@ -0,0 +1,95 @@
+/* Copyright (c) 2011 Wildfire Games
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * adapters for allocators; provides a minimal subset of the
+ * STL allocator interface.
+ */
+
+#ifndef ALLOCATOR_ADAPTERS
+#define ALLOCATOR_ADAPTERS
+
+#include <memory>
+
+#include "lib/sysdep/rtl.h"
+#include "lib/sysdep/vm.h"
+
+// NB: STL allocators are parameterized on the object type and indicate
+// the number of elements to [de]allocate. however, these adapters are
+// only used for allocating storage and receive the number of bytes.
+
+struct Allocator_Heap
+{
+	void* allocate(size_t size)
+	{
+		return malloc(size);
+	}
+
+	void deallocate(void* p, size_t UNUSED(size))
+	{
+		return free(p);
+	}
+};
+
+template<size_t alignment = allocationAlignment>
+struct Allocator_Aligned
+{
+	void* allocate(size_t size)
+	{
+		return rtl_AllocateAligned(size, alignment);
+	}
+
+	void deallocate(void* p, size_t UNUSED(size))
+	{
+		return rtl_FreeAligned(p);
+	}
+};
+
+template<vm::PageType pageType = vm::kDefault, int prot = PROT_READ|PROT_WRITE>
+struct Allocator_VM
+{
+	void* allocate(size_t size)
+	{
+		return vm::Allocate(size, pageType, prot);
+	}
+
+	void deallocate(void* p, size_t size)
+	{
+		vm::Free(p, size);
+	}
+};
+
+template<size_t commitSize = largePageSize, vm::PageType pageType = vm::kDefault, int prot = PROT_READ|PROT_WRITE>
+struct Allocator_AddressSpace
+{
+	void* allocate(size_t size)
+	{
+		return vm::ReserveAddressSpace(size, commitSize, pageType, prot);
+	}
+
+	void deallocate(void* p, size_t size)
+	{
+		vm::ReleaseAddressSpace(p, size);
+	}
+};
+
+#endif	// #ifndef ALLOCATOR_ADAPTERS

source/lib/allocators/allocator_policies.h

@@ -0,0 +1,354 @@
+/* Copyright (c) 2011 Wildfire Games
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * policy class templates for allocators.
+ */
+
+#ifndef ALLOCATOR_POLICIES
+#define ALLOCATOR_POLICIES
+
+#include "lib/alignment.h"	// pageSize
+#include "lib/allocators/allocator_adapters.h"
+#include "lib/allocators/freelist.h"
+
+
+namespace Allocators {
+
+
+//-----------------------------------------------------------------------------
+// Growth
+
+// O(N) allocations, O(1) wasted space.
+template<size_t increment = pageSize>
+struct Growth_Linear
+{
+	size_t operator()(size_t oldSize) const
+	{
+		return oldSize + increment;
+	}
+};
+
+// O(log r) allocations, O(N) wasted space. NB: the common choice of
+// expansion factor r = 2 (e.g. in the GCC STL) prevents
+// Storage_Reallocate from reusing previous memory blocks,
+// thus constantly growing the heap and decreasing locality.
+// Alexandrescu [C++ and Beyond 2010] recommends r < 33/25.
+// we approximate this with a power of two divisor to allow shifting.
+// C++ does allow reference-to-float template parameters, but
+// integer arithmetic is expected to be faster.
+// (Storage_Commit should use 2:1 because it is cheaper to
+// compute and retains power-of-two sizes.)
+template<size_t multiplier = 21, size_t divisor = 16>
+struct Growth_Exponential
+{
+	size_t operator()(size_t oldSize) const
+	{
+		const size_t product = oldSize * multiplier;
+
+		// detect overflow, but allow equality in case oldSize = 0,
+		// which isn't a problem because Storage_Commit::Expand
+		// raises it to requiredCapacity.
+		ASSERT(product >= oldSize);
+
+		return product / divisor;
+	}
+};
+
+
+//-----------------------------------------------------------------------------
+// Storage
+
+// a contiguous region of memory (not just an "array", because
+// allocators such as Arena append variable-sized intervals).
+//
+// we don't store smart pointers because storage usually doesn't need
+// to be copied, and ICC 11 sometimes wasn't able to inline Address().
+struct Storage
+{
+	// @return starting address (alignment depends on the allocator).
+	uintptr_t Address() const;
+
+	// @return size [bytes] of currently accessible memory.
+	size_t Capacity() const;
+
+	// @return largest possible capacity [bytes].
+	size_t MaxCapacity() const;
+
+	// expand Capacity() to at least requiredCapacity (possibly more
+	//   depending on GrowthPolicy).
+	// @param requiredCapacity > Capacity()
+	// @return false and leave Capacity() unchanged if expansion failed,
+	//   which is guaranteed to happen if requiredCapacity > MaxCapacity().
+	bool Expand(size_t requiredCapacity);
+};
+
+
+// allocate once and refuse subsequent expansion.
+template<class Allocator = Allocator_Aligned<> >
+class Storage_Fixed
+{
+	NONCOPYABLE(Storage_Fixed);
+public:
+	Storage_Fixed(size_t size)
+		: maxCapacity(size)
+		, storage(allocator.allocate(maxCapacity))
+	{
+	}
+
+	~Storage_Fixed()
+	{
+		allocator.deallocate(storage, maxCapacity);
+	}
+
+	uintptr_t Address() const
+	{
+		return uintptr_t(storage);
+	}
+
+	size_t Capacity() const
+	{
+		return maxCapacity;
+	}
+
+	size_t MaxCapacity() const
+	{
+		return maxCapacity;
+	}
+
+	bool Expand(size_t UNUSED(requiredCapacity))
+	{
+		return false;
+	}
+
+private:
+	Allocator allocator;
+	size_t maxCapacity;	// must be initialized before storage
+	void* storage;
+};
+
+
+// unlimited expansion by allocating larger storage and copying.
+// (basically equivalent to std::vector, although Growth_Exponential
+// is much more cache and allocator-friendly than the GCC STL)
+template<class Allocator = Allocator_Heap, class GrowthPolicy = Growth_Exponential<> >
+class Storage_Reallocate
+{
+	NONCOPYABLE(Storage_Reallocate);
+public:
+	Storage_Reallocate(size_t initialCapacity)
+		: capacity(initialCapacity)
+		, storage(allocator.allocate(initialCapacity))
+	{
+	}
+
+	~Storage_Reallocate()
+	{
+		allocator.deallocate(storage, capacity);
+	}
+
+	uintptr_t Address() const
+	{
+		return uintptr_t(storage);
+	}
+
+	size_t Capacity() const
+	{
+		return capacity;
+	}
+
+	size_t MaxCapacity() const
+	{
+		return std::numeric_limits<size_t>::max();
+	}
+
+	bool Expand(size_t requiredCapacity)
+	{
+		size_t newCapacity = std::max(requiredCapacity, GrowthPolicy()(capacity));
+		void* newStorage = allocator.allocate(newCapacity);
+		if(!newStorage)
+			return false;
+		memcpy(newStorage, storage, capacity);
+		std::swap(capacity, newCapacity);
+		std::swap(storage, newStorage);
+		allocator.deallocate(newStorage, newCapacity);	// free PREVIOUS storage
+		return true;
+	}
+
+private:
+	Allocator allocator;
+	size_t capacity;	// must be initialized before storage
+	void* storage;
+};
+
+
+// expand up to the limit of the allocated address space by
+// committing physical memory. this avoids copying and
+// reduces wasted physical memory.
+template<class Allocator = Allocator_AddressSpace<>, class GrowthPolicy = Growth_Exponential<2,1> >
+class Storage_Commit
+{
+	NONCOPYABLE(Storage_Commit);
+public:
+	Storage_Commit(size_t maxCapacity_)
+		: maxCapacity(Align<pageSize>(maxCapacity_))	// see Expand
+		, storage(allocator.allocate(maxCapacity))
+		, capacity(0)
+	{
+	}
+
+	~Storage_Commit()
+	{
+		allocator.deallocate(storage, maxCapacity);
+	}
+
+	uintptr_t Address() const
+	{
+		return uintptr_t(storage);
+	}
+
+	size_t Capacity() const
+	{
+		return capacity;
+	}
+
+	size_t MaxCapacity() const
+	{
+		return maxCapacity;
+	}
+
+	bool Expand(size_t requiredCapacity)
+	{
+		size_t newCapacity = std::max(requiredCapacity, GrowthPolicy()(capacity));
+		// reduce the number of expensive commits by accurately
+		// reflecting the actual capacity. this is safe because
+		// we also round up maxCapacity.
+		newCapacity = Align<pageSize>(newCapacity);
+		if(newCapacity > maxCapacity)
+			return false;
+		if(!vm::Commit(Address()+capacity, newCapacity-capacity))
+			return false;
+		capacity = newCapacity;
+		return true;
+	}
+
+private:
+	Allocator allocator;
+	size_t maxCapacity;	// must be initialized before storage
+	void* storage;
+	size_t capacity;
+};
+
+
+// implicitly expand up to the limit of the allocated address space by
+// committing physical memory when a page is first accessed.
+// this is basically equivalent to Storage_Commit with Growth_Linear,
+// except that there is no need to call Expand.
+template<class Allocator = Allocator_AddressSpace<> >
+class Storage_AutoCommit
+{
+	NONCOPYABLE(Storage_AutoCommit);
+public:
+	Storage_AutoCommit(size_t maxCapacity_)
+		: maxCapacity(Align<pageSize>(maxCapacity_))	// match user's expectation
+		, storage(allocator.allocate(maxCapacity))
+	{
+		vm::BeginOnDemandCommits();
+	}
+
+	~Storage_AutoCommit()
+	{
+		vm::EndOnDemandCommits();
+		allocator.deallocate(storage, maxCapacity);
+	}
+
+	uintptr_t Address() const
+	{
+		return uintptr_t(storage);
+	}
+
+	size_t Capacity() const
+	{
+		return maxCapacity;
+	}
+
+	size_t MaxCapacity() const
+	{
+		return maxCapacity;
+	}
+
+	bool Expand(size_t UNUSED(requiredCapacity))
+	{
+		return false;
+	}
+
+private:
+	Allocator allocator;
+	size_t maxCapacity;	// must be initialized before storage
+	void* storage;
+};
+
+
+// reserve and return a pointer to space at the end of storage,
+// expanding it if need be.
+// @param end total number of previously reserved bytes; will be
+//   increased by size if the allocation succeeds.
+// @param size [bytes] to reserve.
+// @return address of allocated space, or 0 if storage is full
+//   and cannot expand any further.
+template<class Storage>
+static inline uintptr_t StorageAppend(Storage& storage, size_t& end, size_t size)
+{
+	size_t newEnd = end + size;
+	if(newEnd > storage.Capacity())
+	{
+		if(!storage.Expand(newEnd))	// NB: may change storage.Address()
+			return 0;
+	}
+
+	std::swap(end, newEnd);
+	return storage.Address() + newEnd;
+}
+
+
+// invoke operator() on default-constructed instantiations of
+// Functor for reasonable combinations of Storage and their parameters.
+template<template<class Storage> class Functor>
+static void ForEachStorage()
+{
+	Functor<Storage_Fixed<Allocator_Heap> >()();
+	Functor<Storage_Fixed<Allocator_Aligned<> > >()();
+
+	Functor<Storage_Reallocate<Allocator_Heap, Growth_Linear<> > >()();
+	Functor<Storage_Reallocate<Allocator_Heap, Growth_Exponential<> > >()();
+	Functor<Storage_Reallocate<Allocator_Aligned<>, Growth_Linear<> > >()();
+	Functor<Storage_Reallocate<Allocator_Aligned<>, Growth_Exponential<> > >()();
+
+	Functor<Storage_Commit<Allocator_AddressSpace<>, Growth_Linear<> > >()();
+	Functor<Storage_Commit<Allocator_AddressSpace<>, Growth_Exponential<> > >()();
+
+	Functor<Storage_AutoCommit<> >()();
+}
+
+}	// namespace Allocators
+
+#endif	// #ifndef ALLOCATOR_POLICIES

source/lib/allocators/arena.h

@@ -0,0 +1,82 @@
+/* Copyright (c) 2010 Wildfire Games
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * arena allocator (variable-size blocks, no deallocation).
+ */
+
+#ifndef INCLUDED_ALLOCATORS_ARENA
+#define INCLUDED_ALLOCATORS_ARENA
+
+#include "lib/allocators/allocator_policies.h"
+
+namespace Allocators {
+
+/**
+ * allocator design parameters:
+ * - O(1) allocation;
+ * - variable-size blocks;
+ * - support for deallocating all objects;
+ * - consecutive allocations are back-to-back;
+ * - no extra alignment nor padding.
+ **/
+template<class Storage>
+class Arena
+{
+public:
+	Arena(size_t maxSize)
+		: storage(maxSize)
+	{
+		DeallocateAll();
+	}
+
+	size_t RemainingBytes() const
+	{
+		return storage.MaxCapacity() - end;
+	}
+
+	void* Allocate(size_t size)
+	{
+		return (void*)StorageAppend(storage, end, size);
+	}
+
+	void DeallocateAll()
+	{
+		end = 0;
+	}
+
+	// @return whether the address lies within the previously allocated range.
+	bool Contains(uintptr_t address) const
+	{
+		return (address - storage.Address()) < end;
+	}
+
+private:
+	Storage storage;
+	size_t end;
+};
+
+LIB_API void TestArena();
+
+}	// namespace Allocators
+
+#endif	// #ifndef INCLUDED_ALLOCATORS_ARENA

source/lib/allocators/dynarray.cpp

@@ -28,7 +28,7 @@
 #include "lib/allocators/dynarray.h"
 
 #include "lib/alignment.h"
-#include "lib/allocators/page_aligned.h"
+#include "lib/sysdep/vm.h"
 
 
 static Status validate_da(DynArray* da)
@@ -39,7 +39,6 @@ static Status validate_da(DynArray* da)
 	const size_t max_size_pa = da->max_size_pa;
 	const size_t cur_size    = da->cur_size;
 	const size_t pos         = da->pos;
-	const int prot           = da->prot;
 
 	// note: this happens if max_size == 0
 //	if(debug_IsPointerBogus(base))
@@ -52,8 +51,6 @@ static Status validate_da(DynArray* da)
 		WARN_RETURN(ERR::_4);
 	if(pos > cur_size || pos > max_size_pa)
 		WARN_RETURN(ERR::_5);
-	if(prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC))
-		WARN_RETURN(ERR::_6);
 
 	return INFO::OK;
 }
@@ -63,17 +60,17 @@ static Status validate_da(DynArray* da)
 
 Status da_alloc(DynArray* da, size_t max_size)
 {
+	ENSURE(max_size != 0);
 	const size_t max_size_pa = Align<pageSize>(max_size);
 
-	u8* p = 0;
-	if(max_size_pa)	// (avoid mmap failure)
-		RETURN_STATUS_IF_ERR(mem_Reserve(max_size_pa, &p));
+	u8* p = (u8*)vm::ReserveAddressSpace(max_size_pa);
+	if(!p)
+		return ERR::NO_MEM;	// NOWARN (already done in vm)
 
 	da->base        = p;
 	da->max_size_pa = max_size_pa;
 	da->cur_size    = 0;
 	da->cur_size_pa = 0;
-	da->prot        = PROT_READ|PROT_WRITE;
 	da->pos         = 0;
 	CHECK_DA(da);
 	return INFO::OK;
@@ -84,15 +81,11 @@ Status da_free(DynArray* da)
 {
 	CHECK_DA(da);
 
-	u8* p            = da->base;
-	size_t size_pa   = da->max_size_pa;
+	vm::ReleaseAddressSpace(da->base, da->max_size_pa);
 
 	// wipe out the DynArray for safety
-	// (must be done here because mem_Release may fail)
 	memset(da, 0, sizeof(*da));
 
-	if(size_pa)
-		RETURN_STATUS_IF_ERR(mem_Release(p, size_pa));
 	return INFO::OK;
 }
 
@@ -113,19 +106,20 @@ Status da_set_size(DynArray* da, size_t new_size)
 		return ERR::LIMIT;	// NOWARN
 
 	u8* end = da->base + cur_size_pa;
+	bool ok = true;
 	// expanding
 	if(size_delta_pa > 0)
-		RETURN_STATUS_IF_ERR(mem_Commit(end, size_delta_pa, da->prot));
+		ok = vm::Commit(uintptr_t(end), size_delta_pa);
 	// shrinking
 	else if(size_delta_pa < 0)
-		RETURN_STATUS_IF_ERR(mem_Decommit(end+size_delta_pa, -size_delta_pa));
+		ok = vm::Decommit(uintptr_t(end+size_delta_pa), -size_delta_pa);
 	// else: no change in page count, e.g. if going from size=1 to 2
 	// (we don't want mem_* to have to handle size=0)
 
 	da->cur_size = new_size;
 	da->cur_size_pa = new_size_pa;
 	CHECK_DA(da);
-	return INFO::OK;
+	return ok? INFO::OK : ERR::FAIL;
 }
 
 
@@ -138,18 +132,6 @@ Status da_reserve(DynArray* da, size_t size)
 }
 
 
-Status da_set_prot(DynArray* da, int prot)
-{
-	CHECK_DA(da);
-
-	da->prot = prot;
-	RETURN_STATUS_IF_ERR(mem_Protect(da->base, da->cur_size_pa, prot));
-
-	CHECK_DA(da);
-	return INFO::OK;
-}
-
-
 Status da_append(DynArray* da, const void* data, size_t size)
 {
 	RETURN_STATUS_IF_ERR(da_reserve(da, size));

source/lib/allocators/dynarray.h

@@ -43,11 +43,6 @@ struct DynArray
 	size_t cur_size;	 /// committed
 	size_t cur_size_pa;
 
-	/**
-	 * mprotect flags applied to newly committed pages
-	 **/
-	int prot;
-
 	size_t pos;
 };
 
@@ -97,18 +92,6 @@ LIB_API Status da_set_size(DynArray* da, size_t new_size);
  **/
 LIB_API Status da_reserve(DynArray* da, size_t size);
 
-/**
- * change access rights of the array memory.
- *
- * used to implement write-protection. affects the currently committed
- * pages as well as all subsequently added pages.
- *
- * @param da DynArray.
- * @param prot a combination of the PROT_* values used with mprotect.
- * @return Status.
- **/
-LIB_API Status da_set_prot(DynArray* da, int prot);
-
 /**
  * "write" to array, i.e. copy from the given buffer.
  *

source/lib/allocators/overrun_protector.h

@@ -24,7 +24,7 @@
 #define INCLUDED_ALLOCATORS_OVERRUN_PROTECTOR
 
 #include "lib/config2.h"	// CONFIG2_ALLOCATORS_OVERRUN_PROTECTION
-#include "lib/allocators/page_aligned.h"
+#include "lib/sysdep/vm.h"
 
 /**
 OverrunProtector wraps an arbitrary object in isolated page(s) and
@@ -54,7 +54,7 @@ template<class T> class OverrunProtector
 	NONCOPYABLE(OverrunProtector);	// const member
 public:
 	OverrunProtector()
-		: object(new(page_aligned_alloc(sizeof(T))) T())
+		: object(new(vm::Allocate(sizeof(T))) T())
 	{
 		lock();
 	}
@@ -63,7 +63,7 @@ public:
 	{
 		unlock();
 		object->~T();	// call dtor (since we used placement new)
-		page_aligned_free(object, sizeof(T));
+		vm::Free(object, sizeof(T));
 	}
 
 	T* get() const
@@ -75,7 +75,7 @@ public:
 	void lock() const
 	{
 #if CONFIG2_ALLOCATORS_OVERRUN_PROTECTION
-		mprotect(object, sizeof(T), PROT_NONE);
+		vm::Protect(object, sizeof(T), PROT_NONE);
 #endif
 	}
 
@@ -83,7 +83,7 @@ private:
 	void unlock() const
 	{
 #if CONFIG2_ALLOCATORS_OVERRUN_PROTECTION
-		mprotect(object, sizeof(T), PROT_READ|PROT_WRITE);
+		vm::Protect(object, sizeof(T), PROT_READ|PROT_WRITE);
 #endif
 	}
 

source/lib/allocators/pool.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -29,11 +29,59 @@
 
 #include "lib/alignment.h"
 #include "lib/allocators/freelist.h"
+#include "lib/allocators/allocator_adapters.h"
 
 #include "lib/timer.h"
 
-TIMER_ADD_CLIENT(tc_pool_alloc);
+namespace Allocators {
 
+template<class Storage>
+struct BasicPoolTest
+{
+	void operator()() const
+	{
+		Pool<double, Storage> p(100);
+		const size_t initialSpace = p.RemainingObjects();
+		double* p1 = p.Allocate();
+		ENSURE(p1 != 0);
+		ENSURE(p.Contains(uintptr_t(p1)));
+		ENSURE(p.RemainingObjects() == initialSpace-1);
+		ENSURE(p.Contains(uintptr_t(p1)+1));
+		ENSURE(p.Contains(uintptr_t(p1)+sizeof(double)-1));
+		ENSURE(!p.Contains(uintptr_t(p1)-1));
+		ENSURE(!p.Contains(uintptr_t(p1)+sizeof(double)));
+		if(p.RemainingObjects() == 0)
+			ENSURE(p.Allocate() == 0);	// full
+		else
+			ENSURE(p.Allocate() != 0);	// can still expand
+		p.DeallocateAll();
+		ENSURE(!p.Contains(uintptr_t(p1)));
+
+		p1 = p.Allocate();
+		ENSURE(p1 != 0);
+		ENSURE(p.Contains(uintptr_t(p1)));
+		ENSURE(p.RemainingObjects() == initialSpace-1);
+		double* p2 = p.Allocate();
+		ENSURE(p2 != 0);
+		ENSURE(p.Contains(uintptr_t(p2)));
+		ENSURE(p.RemainingObjects() == initialSpace-2);
+		ENSURE(p2 == (double*)(uintptr_t(p1)+sizeof(double)));
+		if(p.RemainingObjects() == 0)
+			ENSURE(p.Allocate() == 0);	// full
+		else
+			ENSURE(p.Allocate() != 0);	// can still expand
+	}
+};
+
+void TestPool()
+{
+	ForEachStorage<BasicPoolTest>();
+}
+
+}	// namespace Allocators
+
+
+TIMER_ADD_CLIENT(tc_pool_alloc);
 
 Status pool_create(Pool* p, size_t max_size, size_t el_size)
 {

source/lib/allocators/pool.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -21,12 +21,84 @@
  */
 
 /*
- * pool allocator
+ * pool allocator (fixed-size blocks, freelist).
  */
 
 #ifndef INCLUDED_ALLOCATORS_POOL
 #define INCLUDED_ALLOCATORS_POOL
 
+#include "lib/bits.h"	// ROUND_UP
+#include "lib/allocators/allocator_policies.h"
+
+namespace Allocators {
+	
+/**
+ * allocator design parameters:
+ * - O(1) allocation and deallocation;
+ * - fixed-size objects;
+ * - support for deallocating all objects;
+ * - consecutive allocations are back-to-back;
+ * - objects are aligned to the pointer size.
+ **/
+template<typename T, class Storage = Storage_Fixed<> >
+class Pool
+{
+public:
+	// (must round up because freelist stores pointers inside objects)
+	static const size_t objectSize = ROUND_UP(sizeof(T), sizeof(intptr_t));
+
+	Pool(size_t maxObjects)
+		: storage(maxObjects*objectSize)
+	{
+		DeallocateAll();
+	}
+
+	size_t RemainingObjects()
+	{
+		return (storage.MaxCapacity() - end) / objectSize;
+	}
+
+	T* Allocate()
+	{
+		void* p = mem_freelist_Detach(freelist);
+		if(p)
+		{
+			ASSERT(Contains(p));
+			return (T*)p;
+		}
+
+		return (T*)StorageAppend(storage, end, objectSize);
+	}
+
+	void Deallocate(T* p)
+	{
+		ASSERT(Contains(p));
+		mem_freelist_AddToFront(freelist, p);
+	}
+
+	void DeallocateAll()
+	{
+		freelist = mem_freelist_Sentinel();
+		end = 0;
+	}
+
+	// @return whether the address lies within the previously allocated range.
+	bool Contains(uintptr_t address) const
+	{
+		return (address - storage.Address()) < end;
+	}
+
+private:
+	Storage storage;
+	size_t end;
+	void* freelist;
+};
+
+LIB_API void TestPool();
+
+}	// namespace Allocators
+
+
 #include "lib/allocators/dynarray.h"
 
 /**
@@ -145,43 +217,6 @@ LIB_API void pool_free_all(Pool* p);
 LIB_API size_t pool_committed(Pool* p);
 
 
-/**
- * C++ wrapper on top of pool_alloc for fixed-size allocations (determined by sizeof(T))
- *
- * T must be POD (Plain Old Data) because it is memset to 0!
- **/
-template<class T>
-class PoolAllocator
-{
-public:
-	explicit PoolAllocator(size_t maxElements)
-	{
-		(void)pool_create(&m_pool, maxElements*sizeof(T), sizeof(T));
-	}
-
-	~PoolAllocator()
-	{
-		(void)pool_destroy(&m_pool);
-	}
-
-	T* AllocateZeroedMemory()
-	{
-		T* t = (T*)pool_alloc(&m_pool, 0);
-		if(!t)
-			throw std::bad_alloc();
-		memset(t, 0, sizeof(T));
-		return t;
-	}
-
-	void Free(T* t)
-	{
-		pool_free(&m_pool, t);
-	}
-
-private:
-	Pool m_pool;
-};
-
 /**
  * C++ wrapper on top of pool_alloc for variable-sized allocations.
  * Memory is returned uninitialised.

source/lib/allocators/tests/test_allocators.h

@@ -35,7 +35,6 @@ public:
 		// basic test of functionality (not really meaningful)
 		TS_ASSERT_OK(da_alloc(&da, 1000));
 		TS_ASSERT_OK(da_set_size(&da, 1000));
-		TS_ASSERT_OK(da_set_prot(&da, PROT_NONE));
 		TS_ASSERT_OK(da_free(&da));
 	}
 };

source/lib/allocators/unique_range.cpp

@@ -28,6 +28,8 @@ static UniqueRangeDeleter deleters[allocationAlignment] = { FreeNone, FreeAligne
 static IdxDeleter numDeleters = 2;
 
 
+// NB: callers should skip this if *idxDeleterOut != 0 (avoids the overhead
+// of an unnecessary indirect function call)
 void RegisterUniqueRangeDeleter(UniqueRangeDeleter deleter, volatile IdxDeleter* idxDeleterOut)
 {
 	ENSURE(deleter);
@@ -44,7 +46,7 @@ void RegisterUniqueRangeDeleter(UniqueRangeDeleter deleter, volatile IdxDeleter*
 	ENSURE(idxDeleter < (IdxDeleter)ARRAY_SIZE(deleters));
 	deleters[idxDeleter] = deleter;
 	COMPILER_FENCE;
-	*idxDeleterOut = idxDeleter;	// linearization point
+	*idxDeleterOut = idxDeleter;
 }
 
 
@@ -66,8 +68,20 @@ UniqueRange AllocateAligned(size_t size, size_t alignment)
 	const UniqueRange::pointer p = rtl_AllocateAligned(alignedSize, alignment);
 
 	static volatile IdxDeleter idxDeleterAligned;
-	if(idxDeleterAligned == 0)
+	if(idxDeleterAligned == 0)	// (optional optimization)
 		RegisterUniqueRangeDeleter(FreeAligned, &idxDeleterAligned);
 
 	return RVALUE(UniqueRange(p, size, idxDeleterAligned));
 }
+
+
+UniqueRange AllocateVM(size_t size, vm::PageType pageType, int prot)
+{
+	const UniqueRange::pointer p = vm::Allocate(size, pageType, prot);
+
+	static volatile IdxDeleter idxDeleter;
+	if(idxDeleter == 0)	// (optional optimization)
+		RegisterUniqueRangeDeleter(vm::Free, &idxDeleter);
+
+	return RVALUE(UniqueRange(p, size, idxDeleter));
+}

source/lib/allocators/unique_range.h

@@ -3,6 +3,7 @@
 
 #include "lib/lib_api.h"
 #include "lib/alignment.h"	// allocationAlignment
+#include "lib/sysdep/vm.h"
 
 // we usually don't hold multiple references to allocations, so unique_ptr
 // can be used instead of the more complex (ICC generated incorrect code on
@@ -191,4 +192,7 @@ static inline void swap(UniqueRange& p1, RVALUE_REF(UniqueRange) p2)
 
 LIB_API UniqueRange AllocateAligned(size_t size, size_t alignment);
 
+LIB_API UniqueRange AllocateVM(size_t size, vm::PageType pageSize = vm::kDefault, int prot = PROT_READ|PROT_WRITE);
+
+
 #endif	// #ifndef INCLUDED_ALLOCATORS_UNIQUE_RANGE

source/lib/bits.h

@@ -229,6 +229,10 @@ inline T round_down(T n, T multiple)
 	return result;
 }
 
+// evaluates to an expression suitable as an initializer
+// for constant static data members.
+#define ROUND_UP(n, multiple) (((n) + (multiple)-1) & ~((multiple)-1))
+
 
 template<typename T>
 inline T MaxPowerOfTwoDivisor(T value)

source/lib/debug.cpp

@@ -33,8 +33,8 @@
 
 #include "lib/alignment.h"
 #include "lib/app_hooks.h"
-#include "lib/allocators/page_aligned.h"
 #include "lib/fnv_hash.h"
+#include "lib/sysdep/vm.h"
 #include "lib/sysdep/cpu.h"	// cpu_CAS
 #include "lib/sysdep/sysdep.h"
 
@@ -207,7 +207,7 @@ static const size_t messageSize = 512*KiB;
 
 void debug_FreeErrorMessage(ErrorMessageMem* emm)
 {
-	page_aligned_free(emm->pa_mem, messageSize);
+	vm::Free(emm->pa_mem, messageSize);
 }
 
 
@@ -274,7 +274,7 @@ const wchar_t* debug_BuildErrorMessage(
 	sys_StatusDescription(0, os_error, ARRAY_SIZE(os_error));
 
 	// rationale: see ErrorMessageMem
-	emm->pa_mem = page_aligned_alloc(messageSize);
+	emm->pa_mem = vm::Allocate(messageSize);
 	wchar_t* const buf = (wchar_t*)emm->pa_mem;
 	if(!buf)
 		return L"(insufficient memory to generate error message)";

source/lib/debug.h

@@ -537,8 +537,8 @@ struct ErrorMessageMem
 	// rationale:
 	// - error messages with stack traces require a good deal of memory
 	//   (hundreds of KB). static buffers of that size are undesirable.
- 	// - the heap may be corrupted, so don't use malloc. allocator.h's
-	//   page_aligned_malloc (implemented via mmap) should be safe.
+ 	// - the heap may be corrupted, so don't use malloc.
+	//   instead, "lib/sysdep/vm.h" functions should be safe.
 	// - alloca is a bit iffy (the stack may be maxed out), non-portable and
 	//   complicates the code because it can't be allocated by a subroutine.
 	// - this method is probably slow, but error messages aren't built often.

source/lib/pch/pch_warnings.h

@@ -10,6 +10,7 @@
 // .. always disabled W4
 # pragma warning(disable:4103)	// alignment changed after including header (boost has #pragma pack/pop in separate headers)
 # pragma warning(disable:4127)	// conditional expression is constant; rationale: see STMT in lib.h.
+# pragma warning(disable:4324)	// structure was padded due to __declspec(align())
 # pragma warning(disable:4351)	// yes, default init of array entries is desired
 # pragma warning(disable:4355)	// 'this' used in base member initializer list
 # pragma warning(disable:4718)	// recursive call has no side effects, deleting

source/lib/sysdep/os/unix/uvm.cpp

@@ -0,0 +1,136 @@
+/* Copyright (c) 2011 Wildfire Games
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "precompiled.h"
+#include "lib/sysdep/vm.h"
+
+#include "lib/alignment.h"
+
+// "anonymous" effectively means mapping /dev/zero, but is more efficient.
+// MAP_ANONYMOUS is not in SUSv3, but is a very common extension.
+// unfortunately, MacOS X only defines MAP_ANON, which Solaris says is
+// deprecated. workaround there: define MAP_ANONYMOUS in terms of MAP_ANON.
+#ifndef MAP_ANONYMOUS
+# define MAP_ANONYMOUS MAP_ANON
+#endif
+
+static const int mmap_flags = MAP_PRIVATE|MAP_ANONYMOUS;
+
+namespace vm {
+
+void* ReserveAddressSpace(size_t size, size_t UNUSED(commitSize), PageType UNUSED(pageType), int UNUSED(prot))
+{
+	errno = 0;
+	void* p = mmap(0, size, PROT_NONE, mmap_flags|MAP_NORESERVE, -1, 0);
+	if(p == MAP_FAILED)
+		return 0;
+	return p;
+}
+
+void ReleaseAddressSpace(void* p, size_t size)
+{
+	ENSURE(size != 0);
+
+	errno = 0;
+	if(munmap(p, size) != 0)
+		DEBUG_WARN_ERR(StatusFromErrno());
+}
+
+
+bool Commit(uintptr_t address, size_t size, PageType UNUSED(pageType), int prot)
+{
+	if(prot == PROT_NONE)	// would be understood as a request to decommit
+	{
+		DEBUG_WARN_ERR(ERR::INVALID_PARAM);
+		return false;
+	}
+
+	errno = 0;
+	if(mmap((void*)address, size, prot, mmap_flags|MAP_FIXED, -1, 0) == MAP_FAILED)
+		return false;
+
+	if(prot != (PROT_READ|PROT_WRITE))
+		(void)Protect(address, size, prot);
+
+	return true;
+}
+
+bool Decommit(void* p, size_t size)
+{
+	errno = 0;
+	if(mmap(p, size, PROT_NONE, mmap_flags|MAP_NORESERVE|MAP_FIXED, -1, 0) == MAP_FAILED)
+		return false;
+	return true;
+}
+
+
+bool Protect(uintptr_t address, size_t size, int prot)
+{
+	errno = 0;
+	if(mprotect((void*)address, size, prot) != 0)
+	{
+		DEBUG_WARN_ERR(ERR::FAIL);
+		return false;
+	}
+	return true;
+}
+
+
+void* Allocate(size_t size, PageType pageType, int prot)
+{
+	void* p = ReserveAddressSpace(size);
+	if(!p)
+		return 0;
+
+	if(!Commit(uintptr_t(p), size, pageType, prot))
+	{
+		ReleaseAddressSpace(p, size);
+		return 0;
+	}
+
+	return p;
+}
+
+void Free(void* p, size_t size)
+{
+	// (only the Windows implementation distinguishes between Free and ReleaseAddressSpace)
+	vm::ReleaseAddressSpace(p, size);
+}
+
+
+void BeginOnDemandCommits()
+{
+	// not yet implemented, but possible with a signal handler
+}
+
+void EndOnDemandCommits()
+{
+	// not yet implemented, but possible with a signal handler
+}
+
+
+void DumpStatistics()
+{
+	// we haven't collected any statistics
+}
+
+}	// namespace vm

source/lib/sysdep/os/win/wnuma.cpp

@@ -27,9 +27,9 @@
 #include "lib/alignment.h"
 #include "lib/timer.h"
 #include "lib/module_init.h"
-#include "lib/allocators/page_aligned.h"
-#include "lib/sysdep/os_cpu.h"
+#include "lib/sysdep/vm.h"
 #include "lib/sysdep/acpi.h"
+#include "lib/sysdep/os_cpu.h"
 #include "lib/sysdep/os/win/win.h"
 #include "lib/sysdep/os/win/wutil.h"
 #include "lib/sysdep/os/win/wcpu.h"
@@ -374,7 +374,7 @@ static double ReadRelativeDistanceFromSLIT(const SLIT* slit)
 static double MeasureRelativeDistance()
 {
 	const size_t size = 32*MiB;
-	void* mem = page_aligned_alloc(size);
+	void* mem = vm::Allocate(size);
 	ASSUME_ALIGNED(mem, pageSize);
 
 	const uintptr_t previousProcessorMask = os_cpu_SetThreadAffinityMask(os_cpu_ProcessorMask());
@@ -395,7 +395,7 @@ static double MeasureRelativeDistance()
 
 	(void)os_cpu_SetThreadAffinityMask(previousProcessorMask);
 
-	page_aligned_free(mem, size);
+	vm::Free(mem, size);
 
 	return maxTime / minTime;
 }
@@ -462,81 +462,54 @@ bool numa_IsMemoryInterleaved()
 
 
 //-----------------------------------------------------------------------------
-// allocator
-//
-//static bool VerifyPages(void* mem, size_t size, size_t pageSize, size_t node)
-//{
-//	WUTIL_FUNC(pQueryWorkingSetEx, BOOL, (HANDLE, PVOID, DWORD));
-//	WUTIL_IMPORT_KERNEL32(QueryWorkingSetEx, pQueryWorkingSetEx);
-//	if(!pQueryWorkingSetEx)
-//		return true;	// can't do anything
-//
-//#if WINVER >= 0x600
-//	size_t largePageSize = os_cpu_LargePageSize();
-//	ENSURE(largePageSize != 0); // this value is needed for later
-//
-//	// retrieve attributes of all pages constituting mem
-//	const size_t numPages = (size + pageSize-1) / pageSize;
-//	PSAPI_WORKING_SET_EX_INFORMATION* wsi = new PSAPI_WORKING_SET_EX_INFORMATION[numPages];
-//	for(size_t i = 0; i < numPages; i++)
-//		wsi[i].VirtualAddress = (u8*)mem + i*pageSize;
-//	pQueryWorkingSetEx(GetCurrentProcess(), wsi, DWORD(sizeof(PSAPI_WORKING_SET_EX_INFORMATION)*numPages));
-//
-//	// ensure each is valid and allocated on the correct node
-//	for(size_t i = 0; i < numPages; i++)
-//	{
-//		const PSAPI_WORKING_SET_EX_BLOCK& attributes = wsi[i].VirtualAttributes;
-//		if(!attributes.Valid)
-//			return false;
-//		if((attributes.LargePage != 0) != (pageSize == largePageSize))
-//		{
-//			debug_printf(L"NUMA: is not a large page\n");
-//			return false;
-//		}
-//		if(attributes.Node != node)
-//		{
-//			debug_printf(L"NUMA: allocated from remote node\n");
-//			return false;
-//		}
-//	}
-//
-//	delete[] wsi;
-//#else
-//	UNUSED2(mem);
-//	UNUSED2(size);
-//	UNUSED2(pageSize);
-//	UNUSED2(node);
-//#endif
-//
-//	return true;
-//}
-//
-//
-//void* numa_AllocateOnNode(size_t node, size_t size, LargePageDisposition largePageDisposition, size_t* ppageSize)
-//{
-//	ENSURE(node < numa_NumNodes());
-//
-//	// see if there will be enough memory (non-authoritative, for debug purposes only)
-//	{
-//		const size_t sizeMiB = size/MiB;
-//		const size_t availableMiB = numa_AvailableMemory(node);
-//		if(availableMiB < sizeMiB)
-//			debug_printf(L"NUMA: warning: node reports insufficient memory (%d vs %d MB)\n", availableMiB, sizeMiB);
-//	}
-//
-//	size_t pageSize;	// (used below even if ppageSize is zero)
-//	void* const mem = numa_Allocate(size, largePageDisposition, &pageSize);
-//	if(ppageSize)
-//		*ppageSize = pageSize;
-//
-//	// we can't use VirtualAllocExNuma - it's only available in Vista and Server 2008.
-//	// workaround: fault in all pages now to ensure they are allocated from the
-//	// current node, then verify page attributes.
-//	const uintptr_t previousProcessorMask = os_cpu_SetThreadAffinityMask(numa_ProcessorMaskFromNode(node));
-//	memset(mem, 0, size);
-//	(void)os_cpu_SetThreadAffinityMask(previousProcessorMask);
-//
-//	VerifyPages(mem, size, pageSize, node);
-//
-//	return mem;
-//}
+
+#if 0
+
+static bool VerifyPages(void* mem, size_t size, size_t pageSize, size_t node)
+{
+	WUTIL_FUNC(pQueryWorkingSetEx, BOOL, (HANDLE, PVOID, DWORD));
+	WUTIL_IMPORT_KERNEL32(QueryWorkingSetEx, pQueryWorkingSetEx);
+	if(!pQueryWorkingSetEx)
+		return true;	// can't do anything
+
+#if WINVER >= 0x600
+	size_t largePageSize = os_cpu_LargePageSize();
+	ENSURE(largePageSize != 0); // this value is needed for later
+
+	// retrieve attributes of all pages constituting mem
+	const size_t numPages = (size + pageSize-1) / pageSize;
+	PSAPI_WORKING_SET_EX_INFORMATION* wsi = new PSAPI_WORKING_SET_EX_INFORMATION[numPages];
+	for(size_t i = 0; i < numPages; i++)
+		wsi[i].VirtualAddress = (u8*)mem + i*pageSize;
+	pQueryWorkingSetEx(GetCurrentProcess(), wsi, DWORD(sizeof(PSAPI_WORKING_SET_EX_INFORMATION)*numPages));
+
+	// ensure each is valid and allocated on the correct node
+	for(size_t i = 0; i < numPages; i++)
+	{
+		const PSAPI_WORKING_SET_EX_BLOCK& attributes = wsi[i].VirtualAttributes;
+		if(!attributes.Valid)
+			return false;
+		if((attributes.LargePage != 0) != (pageSize == largePageSize))
+		{
+			debug_printf(L"NUMA: is not a large page\n");
+			return false;
+		}
+		if(attributes.Node != node)
+		{
+			debug_printf(L"NUMA: allocated from remote node\n");
+			return false;
+		}
+	}
+
+	delete[] wsi;
+#else
+	UNUSED2(mem);
+	UNUSED2(size);
+	UNUSED2(pageSize);
+	UNUSED2(node);
+#endif
+
+	return true;
+}
+
+#endif

source/lib/sysdep/os/win/wposix/wmman.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -27,20 +27,14 @@
 #include "lib/sysdep/os/win/wposix/crt_posix.h"		// _get_osfhandle
 
 
-//-----------------------------------------------------------------------------
-// memory mapping
-//-----------------------------------------------------------------------------
-
-// convert POSIX PROT_* flags to their Win32 PAGE_* enumeration equivalents.
-// used by mprotect.
-static DWORD win32_prot(int prot)
+unsigned MemoryProtectionFromPosix(int prot)
 {
+	if(prot == PROT_NONE)
+		return PAGE_NOACCESS;
+
 	// this covers all 8 combinations of read|write|exec
-	// (note that "none" means all flags are 0).
 	switch(prot & (PROT_READ|PROT_WRITE|PROT_EXEC))
 	{
-	case PROT_NONE:
-		return PAGE_NOACCESS;
 	case PROT_READ:
 		return PAGE_READONLY;
 	case PROT_WRITE:
@@ -57,15 +51,22 @@ static DWORD win32_prot(int prot)
 		return PAGE_EXECUTE_READWRITE;
 	case PROT_READ|PROT_WRITE|PROT_EXEC:
 		return PAGE_EXECUTE_READWRITE;
+	default:	// none set
+		DEBUG_WARN_ERR(ERR::INVALID_FLAG);
+		return PAGE_NOACCESS;
 	}
 
-	return 0;	// UNREACHABLE
+	// UNREACHABLE
 }
 
 
+//-----------------------------------------------------------------------------
+// memory mapping
+//-----------------------------------------------------------------------------
+
 int mprotect(void* addr, size_t len, int prot)
 {
-	const DWORD newProtect = win32_prot(prot);
+	const DWORD newProtect = (DWORD)MemoryProtectionFromPosix(prot);
 	DWORD oldProtect;	// required by VirtualProtect
 	const BOOL ok = VirtualProtect(addr, len, newProtect, &oldProtect);
 	WARN_IF_FALSE(ok);
@@ -104,7 +105,7 @@ static Status mmap_mem(void* start, size_t len, int prot, int flags, int fd, voi
 	}
 
 	const DWORD allocationType = want_commit? MEM_COMMIT : MEM_RESERVE;
-	const DWORD protect = win32_prot(prot);
+	const DWORD protect = (DWORD)MemoryProtectionFromPosix(prot);
 	void* p = VirtualAlloc(start, len, allocationType, protect);
 	if(!p)
 	{

source/lib/sysdep/os/win/wposix/wmman.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -56,4 +56,7 @@ extern int munmap(void* start, size_t len);
 
 extern int mprotect(void* addr, size_t len, int prot); 
 
+// convert POSIX PROT_* flags to their Win32 PAGE_* enumeration equivalents.
+LIB_API unsigned MemoryProtectionFromPosix(int prot);
+
 #endif	// #ifndef INCLUDED_WMMAN

source/lib/sysdep/os/win/wvm.cpp

@@ -0,0 +1,539 @@
+/* Copyright (c) 2011 Wildfire Games
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * virtual memory interface. supercedes POSIX mmap; provides support for
+ * large pages, autocommit, and specifying protection flags during allocation.
+ */
+
+#include "precompiled.h"
+#include "lib/sysdep/vm.h"
+
+#include "lib/sysdep/os/win/wutil.h"
+#include <excpt.h>
+
+#include "lib/timer.h"
+#include "lib/bits.h"	// round_down
+#include "lib/alignment.h"	// CACHE_ALIGNED
+#include "lib/module_init.h"
+#include "lib/sysdep/cpu.h"    // cpu_AtomicAdd
+#include "lib/sysdep/numa.h"
+#include "lib/sysdep/arch/x86_x64/x86_x64.h"	// x86_x64_ApicId
+#include "lib/sysdep/arch/x86_x64/topology.h"	// cpu_topology_ProcessorFromApicId
+#include "lib/sysdep/os/win/wversion.h"
+#include "lib/sysdep/os/win/winit.h"
+WINIT_REGISTER_CRITICAL_INIT(wvm_Init);
+
+
+//-----------------------------------------------------------------------------
+// functions not supported by 32-bit Windows XP
+
+static WUTIL_FUNC(pGetCurrentProcessorNumber, DWORD, (VOID));
+static WUTIL_FUNC(pGetNumaProcessorNode, BOOL, (UCHAR, PUCHAR));
+static WUTIL_FUNC(pVirtualAllocExNuma, LPVOID, (HANDLE, LPVOID, SIZE_T, DWORD, DWORD, DWORD));
+
+static DWORD WINAPI EmulateGetCurrentProcessorNumber(VOID)
+{
+	const u8 apicId = x86_x64_ApicId();
+	const DWORD processor = cpu_topology_ProcessorFromApicId(apicId);
+	ASSERT(processor < os_cpu_MaxProcessors);
+	return processor;
+}
+
+static BOOL WINAPI EmulateGetNumaProcessorNode(UCHAR UNUSED(processor), PUCHAR node)
+{
+	// given that the system doesn't support GetNumaProcessorNode,
+	// it will also lack VirtualAllocExNuma, so the node value we assign
+	// is ignored by EmulateVirtualAllocExNuma.
+	*node = 0;
+	return TRUE;
+}
+
+static LPVOID WINAPI EmulateVirtualAllocExNuma(HANDLE UNUSED(hProcess), LPVOID p, SIZE_T size, DWORD allocationType, DWORD protect, DWORD UNUSED(node))
+{
+	return VirtualAlloc(p, size, allocationType, protect);
+}
+
+
+static Status wvm_Init()
+{
+	WUTIL_IMPORT_KERNEL32(GetCurrentProcessorNumber, pGetCurrentProcessorNumber);
+	WUTIL_IMPORT_KERNEL32(GetNumaProcessorNode, pGetNumaProcessorNode);
+	WUTIL_IMPORT_KERNEL32(VirtualAllocExNuma, pVirtualAllocExNuma);
+
+	if(!pGetCurrentProcessorNumber)
+		pGetCurrentProcessorNumber = &EmulateGetCurrentProcessorNumber;
+	if(!pGetNumaProcessorNode)
+		pGetNumaProcessorNode = &EmulateGetNumaProcessorNode;
+	if(!pVirtualAllocExNuma)
+		pVirtualAllocExNuma = &EmulateVirtualAllocExNuma;
+
+	return INFO::OK;
+}
+
+
+namespace vm {
+
+
+//-----------------------------------------------------------------------------
+// per-processor statistics
+
+// (alignment avoids false sharing)
+CACHE_ALIGNED(struct Statistics)	// POD
+{
+	// thread-safe (required due to concurrent commits)
+	void NotifyLargePageCommit()
+	{
+		cpu_AtomicAdd(&largePageCommits, +1);
+	}
+
+	void NotifySmallPageCommit()
+	{
+		cpu_AtomicAdd(&smallPageCommits, +1);
+	}
+
+	intptr_t largePageCommits;
+	intptr_t smallPageCommits;
+};
+static CACHE_ALIGNED(Statistics) statistics[os_cpu_MaxProcessors];
+
+void DumpStatistics()
+{
+	ENSURE(IsAligned(&statistics[0], cacheLineSize));
+	ENSURE(IsAligned(&statistics[1], cacheLineSize));
+
+	size_t smallPageCommits = 0;
+	size_t largePageCommits = 0;
+	uintptr_t processorsWithNoCommits = 0;
+	for(size_t processor = 0; processor < os_cpu_NumProcessors(); processor++)
+	{
+		const Statistics& s = statistics[processor];
+		if(s.smallPageCommits == 0 && s.largePageCommits == 0)
+			processorsWithNoCommits |= Bit<uintptr_t>(processor);
+		smallPageCommits += s.smallPageCommits;
+		largePageCommits += s.largePageCommits;
+	}
+
+	const size_t totalCommits = smallPageCommits+largePageCommits;
+	if(totalCommits == 0)	// this module wasn't used => don't print debug output
+		return;
+
+	const size_t largePageRatio = totalCommits? largePageCommits*100/totalCommits : 0;
+	debug_printf(L"%d commits (%d, i.e. %d%% of them via large pages)\n", totalCommits, largePageCommits, largePageRatio);
+	if(processorsWithNoCommits != 0)
+		debug_printf(L"  processors with no commits: %x\n", processorsWithNoCommits);
+
+	if(numa_NumNodes() > 1)
+		debug_printf(L"NUMA factor: %.2f\n", numa_Factor());
+}
+
+
+//-----------------------------------------------------------------------------
+// allocator with large-page and NUMA support
+
+static bool largePageAllocationTookTooLong = false;
+
+static bool ShouldUseLargePages(size_t allocationSize, DWORD allocationType, PageType pageType)
+{
+	// don't even check for large page support.
+	if(pageType == kSmall)
+		return false;
+
+	// can't use large pages when reserving - VirtualAlloc would fail with
+	// ERROR_INVALID_PARAMETER.
+	if((allocationType & MEM_COMMIT) == 0)
+		return false;
+
+	// OS lacks support for large pages.
+	if(os_cpu_LargePageSize() == 0)
+		return false;
+
+	// large pages are available and application wants them used.
+	if(pageType == kLarge)
+		return true;
+
+	// default: use a heuristic.
+	{
+		// internal fragmentation would be excessive.
+		if(allocationSize <= largePageSize/2)
+			return false;
+
+		// a previous attempt already took too long.
+		if(largePageAllocationTookTooLong)
+			return false;
+
+		// pre-Vista Windows OSes attempt to cope with page fragmentation by
+		// trimming the working set of all processes, thus swapping them out,
+		// and waiting for contiguous regions to appear. this is terribly
+		// slow (multiple seconds), hence the following heuristic:
+		if(wversion_Number() < WVERSION_VISTA)
+		{
+			// if there's not plenty of free memory, then memory is surely
+			// already fragmented.
+			if(os_cpu_MemoryAvailable() < 2000)	// 2 GB
+				return false;
+		}
+	}
+
+	return true;
+}
+
+
+// used for reserving address space, committing pages, or both.
+static void* AllocateLargeOrSmallPages(uintptr_t address, size_t size, DWORD allocationType, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE)
+{
+	const HANDLE hProcess = GetCurrentProcess();
+	const DWORD protect = MemoryProtectionFromPosix(prot);
+
+	UCHAR node;
+	const DWORD processor = pGetCurrentProcessorNumber();
+	WARN_IF_FALSE(pGetNumaProcessorNode((UCHAR)processor, &node));
+
+	if(ShouldUseLargePages(size, allocationType, pageType))
+	{
+		// MEM_LARGE_PAGES requires aligned addresses and sizes
+		const size_t largePageSize = os_cpu_LargePageSize();
+		const uintptr_t alignedAddress = round_down(address, largePageSize);
+		const size_t alignedSize = round_up(size+largePageSize-1, largePageSize);
+		// note: this call can take SECONDS, which is why several checks are
+		// undertaken before we even try. these aren't authoritative, so we
+		// at least prevent future attempts if it takes too long.
+		const double startTime = timer_Time(); COMPILER_FENCE;
+		void* largePages = pVirtualAllocExNuma(hProcess, LPVOID(alignedAddress), alignedSize, allocationType|MEM_LARGE_PAGES, protect, node);
+		const double elapsedTime = timer_Time() - startTime; COMPILER_FENCE;
+		if(elapsedTime > 0.5)
+			largePageAllocationTookTooLong = true;	// avoid large pages next time
+		if(largePages)
+		{
+			if((allocationType & MEM_COMMIT) != 0)
+				statistics[processor].NotifyLargePageCommit();
+			return largePages;
+		}
+	}
+
+	// try (again) with regular pages
+	void* smallPages = pVirtualAllocExNuma(hProcess, LPVOID(address), size, allocationType, protect, node);
+	if(smallPages)
+	{
+		if((allocationType & MEM_COMMIT) != 0)
+			statistics[processor].NotifySmallPageCommit();
+		return smallPages;
+	}
+
+	return 0;
+}
+
+
+//-----------------------------------------------------------------------------
+// address space reservation
+
+// indicates the extent of a range of address space,
+// and the parameters for committing large/small pages in it.
+//
+// this bookkeeping information increases the safety of on-demand commits,
+// enables different parameters for separate allocations, and allows
+// variable alignment because it retains the original base address.
+// (storing this information within the allocated memory would
+// require mapping an additional page and may waste an entire
+// large page if the base address happens to be aligned already.)
+CACHE_ALIGNED(struct AddressRangeDescriptor)	// POD
+{
+	// attempt to activate this descriptor and reserve address space.
+	// side effect: initializes all fields if successful.
+	//
+	// @param size, commitSize, pageType, prot - see ReserveAddressSpace.
+	// @return INFO::SKIPPED if this descriptor is already in use,
+	//   INFO::OK on success, otherwise ERR::NO_MEM (after showing an
+	//   error message).
+	Status Allocate(size_t size, size_t commitSize, PageType pageType, int prot)
+	{
+		// if this descriptor wasn't yet in use, mark it as busy
+		// (double-checking is cheaper than cpu_CAS)
+		if(base != 0 || !cpu_CAS(&base, intptr_t(0), intptr_t(this)))
+			return INFO::SKIPPED;
+
+		ENSURE(size != 0);		// probably indicates a bug in caller
+		ENSURE((commitSize % largePageSize) == 0 || pageType == kSmall);
+		ASSERT(pageType == kLarge || pageType == kSmall || pageType == kDefault);
+		ASSERT(prot == PROT_NONE || (prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC)) == 0);
+		this->commitSize = commitSize;
+		this->pageType = pageType;
+		this->prot = prot;
+
+		alignment = (pageType == kSmall)? pageSize : largePageSize;
+		totalSize = round_up(size+alignment-1, alignment);
+
+		// NB: it is meaningless to ask for large pages when reserving
+		// (see ShouldUseLargePages). pageType only affects subsequent commits.
+		base = (intptr_t)AllocateLargeOrSmallPages(0, totalSize, MEM_RESERVE);
+		if(!base)
+		{
+			DEBUG_DISPLAY_ERROR(ErrorString());
+			return ERR::NO_MEM;	// NOWARN (error string is more helpful)
+		}
+
+		alignedBase = round_up(uintptr_t(base), alignment);
+		alignedEnd = alignedBase + round_up(size, alignment);
+		return INFO::OK;
+	}
+
+	void Free()
+	{
+		vm::Free((void*)base, totalSize);
+		alignment = alignedBase = alignedEnd = 0;
+		totalSize = 0;
+		COMPILER_FENCE;
+		base = 0;	// release descriptor for subsequent reuse
+	}
+
+	bool Contains(uintptr_t address) const
+	{
+		// safety check: we should never see pointers in the no-man's-land
+		// between the original and rounded up base addresses.
+		ENSURE(!(uintptr_t(base) <= address && address < alignedBase));
+
+		return (alignedBase <= address && address < alignedEnd);
+	}
+
+	bool Commit(uintptr_t address)
+	{
+		// (safe because Allocate rounded up to alignment)
+		const uintptr_t alignedAddress = round_down(address, alignment);
+		ENSURE(alignedBase <= alignedAddress && alignedAddress+commitSize <= alignedEnd);
+		return vm::Commit(alignedAddress, commitSize, pageType, prot);
+	}
+
+	// corresponds to the respective page size (Windows requires
+	// naturally aligned addresses and sizes when committing large pages).
+	// note that VirtualAlloc's alignment defaults to 64 KiB.
+	uintptr_t alignment;
+
+	uintptr_t alignedBase;	// multiple of alignment
+	uintptr_t alignedEnd;	// "
+
+	// (actual requested size / allocated address is required by
+	// ReleaseAddressSpace due to variable alignment.)
+	volatile intptr_t base;	// (type is dictated by cpu_CAS)
+	size_t totalSize;
+
+	// parameters to be relayed to vm::Commit
+	size_t commitSize;
+	PageType pageType;
+	int prot;
+
+//private:
+	static const wchar_t* ErrorString()
+	{
+#if ARCH_IA32
+		return L"Out of address space (64-bit OS is required)";
+#elif OS_WIN
+		// because early AMD64 lacked CMPXCHG16B, the Windows lock-free slist
+		// must squeeze the address, ABA tag and list length (a questionable
+		// design decision) into 64 bits. that leaves 39 bits for the
+		// address, plus 4 implied zero bits due to 16-byte alignment.
+		// [http://www.alex-ionescu.com/?p=50]
+		return L"Out of address space (Windows only provides 8 TiB)";
+#else
+		return L"Out of address space";
+#endif
+	}
+};
+
+// (array size governs the max. number of extant allocations)
+static AddressRangeDescriptor ranges[2*os_cpu_MaxProcessors];
+
+
+static AddressRangeDescriptor* FindDescriptor(uintptr_t address)
+{
+	for(size_t idxRange = 0; idxRange < ARRAY_SIZE(ranges); idxRange++)
+	{
+		AddressRangeDescriptor& d = ranges[idxRange];
+		if(d.Contains(address))
+			return &d;
+	}
+
+	return 0;	// not contained in any allocated ranges
+}
+
+
+void* ReserveAddressSpace(size_t size, size_t commitSize, PageType pageType, int prot)
+{
+	for(size_t idxRange = 0; idxRange < ARRAY_SIZE(ranges); idxRange++)
+	{
+		Status ret = ranges[idxRange].Allocate(size, commitSize, pageType, prot);
+		if(ret == INFO::OK)
+			return (void*)ranges[idxRange].alignedBase;
+		if(ret == ERR::NO_MEM)
+			return 0;
+		// else: descriptor already in use, try the next one
+	}
+
+	// all descriptors are in use; ranges[] was too small
+	DEBUG_WARN_ERR(ERR::LIMIT);
+	return 0;
+}
+
+
+void ReleaseAddressSpace(void* p, size_t UNUSED(size))
+{
+	// it is customary to ignore null pointers
+	if(!p)
+		return;
+
+	AddressRangeDescriptor* d = FindDescriptor(uintptr_t(p));
+	if(d)
+		d->Free();
+	else
+	{
+		debug_printf(L"No AddressRangeDescriptor contains %P\n", p);
+		ENSURE(0);
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// commit/decommit, allocate/free, protect
+
+// [23 page faults for an 8 MPixel image total 789 kc, i.e. < 1 ms]
+TIMER_ADD_CLIENT(tc_commit);
+
+bool Commit(uintptr_t address, size_t size, PageType pageType, int prot)
+{
+	TIMER_ACCRUE_ATOMIC(tc_commit);
+
+	return AllocateLargeOrSmallPages(address, size, MEM_COMMIT, pageType, prot) != 0;
+}
+
+
+bool Decommit(uintptr_t address, size_t size)
+{
+	return VirtualFree(LPVOID(address), size, MEM_DECOMMIT) != FALSE;
+}
+
+
+bool Protect(uintptr_t address, size_t size, int prot)
+{
+	const DWORD protect = MemoryProtectionFromPosix(prot);
+	DWORD oldProtect;	// required by VirtualProtect
+	const BOOL ok = VirtualProtect(LPVOID(address), size, protect, &oldProtect);
+	return ok != FALSE;
+}
+
+
+void* Allocate(size_t size, PageType pageType, int prot)
+{
+	return AllocateLargeOrSmallPages(0, size, MEM_RESERVE|MEM_COMMIT, pageType, prot);
+}
+
+
+void Free(void* p, size_t UNUSED(size))
+{
+	if(p)	// otherwise, VirtualFree complains
+	{
+		const BOOL ok = VirtualFree(p, 0, MEM_RELEASE);
+		WARN_IF_FALSE(ok);
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// on-demand commit
+
+static LONG CALLBACK VectoredHandler(const PEXCEPTION_POINTERS ep)
+{
+	const PEXCEPTION_RECORD er = ep->ExceptionRecord;
+
+	// we only want to handle access violations. (strictly speaking,
+	// unmapped memory causes page faults, but Windows reports them
+	// with EXCEPTION_ACCESS_VIOLATION.)
+	if(er->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
+		return EXCEPTION_CONTINUE_SEARCH;
+
+	// NB: read exceptions are legitimate and occur when updating an
+	// accumulator for the first time.
+
+	// get the source/destination of the read/write operation that
+	// failed. (NB: don't use er->ExceptionAddress - that's the
+	// location of the code that encountered the fault)
+	const uintptr_t address = (uintptr_t)er->ExceptionInformation[1];
+
+	// if unknown (e.g. access violation in kernel address space or
+	// violation of alignment requirements), we don't want to handle it.
+	if(address == ~uintptr_t(0))	
+		return EXCEPTION_CONTINUE_SEARCH;
+
+	// the address space must have been allocated by ReserveAddressSpace
+	// (otherwise we wouldn't know the desired commitSize/pageType/prot).
+	AddressRangeDescriptor* d = FindDescriptor(address);
+	if(!d)
+		return EXCEPTION_CONTINUE_SEARCH;
+
+	// NB: the first access to a page isn't necessarily at offset 0
+	// (memcpy isn't guaranteed to copy sequentially). rounding down
+	// is safe and necessary - see AddressRangeDescriptor::alignment.
+	const uintptr_t alignedAddress = round_down(address, d->alignment);
+	bool ok = d->Commit(alignedAddress);
+	if(!ok)
+	{
+		debug_printf(L"VectoredHandler: Commit(0x%p) failed; address=0x%p\n", alignedAddress, address);
+		ENSURE(0);
+		return EXCEPTION_CONTINUE_SEARCH;
+	}
+
+	// continue at (i.e. retry) the same instruction.
+	return EXCEPTION_CONTINUE_EXECUTION;
+}
+
+
+static PVOID handler;
+static ModuleInitState initState;
+static volatile intptr_t references = 0;	// atomic
+
+static Status InitHandler()
+{
+	ENSURE(handler == 0);
+	handler = AddVectoredExceptionHandler(TRUE, VectoredHandler);
+	ENSURE(handler != 0);
+	return INFO::OK;
+}
+
+static void ShutdownHandler()
+{
+	ENSURE(handler != 0);
+	const ULONG ret = RemoveVectoredExceptionHandler(handler);
+	ENSURE(ret != 0);
+	handler = 0;
+}
+
+void BeginOnDemandCommits()
+{
+	ModuleInit(&initState, InitHandler);
+	cpu_AtomicAdd(&references, +1);
+}
+
+void EndOnDemandCommits()
+{
+	if(cpu_AtomicAdd(&references, -1) == 1)
+		ModuleShutdown(&initState, ShutdownHandler);
+}
+
+}	// namespace vm

source/lib/sysdep/vm.h

@@ -0,0 +1,154 @@
+/* Copyright (c) 2011 Wildfire Games
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * virtual memory interface. supercedes POSIX mmap; provides support for
+ * large pages, autocommit, and specifying protection flags during allocation.
+ */
+
+#ifndef INCLUDED_SYSDEP_VM
+#define INCLUDED_SYSDEP_VM
+
+#include "lib/posix/posix_mman.h"	// PROT_*
+
+namespace vm {
+
+// committing large pages (2 MiB) instead of regular 4 KiB pages can
+// increase TLB coverage and reduce misses for sequential access patterns.
+// however, small page TLBs have more entries, making them better suited
+// to random accesses. it may also take a long time to find/free up
+// contiguous regions of physical memory for large pages. applications
+// can express their preference or go along with the default,
+// which depends on several factors such as allocation size.
+enum PageType
+{
+	kLarge,     // use large if available
+	kSmall,     // always use small
+	kDefault	// heuristic
+};
+
+/**
+ * reserve address space and set the parameters for any later
+ * on-demand commits.
+ *
+ * @param size desired number of bytes. any additional space
+ *   in the last page is also accessible.
+ * @param commitSize [bytes] how much to commit each time.
+ *   larger values reduce the number of page faults at the cost of
+ *   additional internal fragmentation. must be a multiple of
+ *   largePageSize unless pageType == kSmall.
+ * @param pageType chooses between large/small pages for commits.
+ * @param prot memory protection flags for newly committed pages.
+ * @return base address (aligned to the respective page size) or
+ *   0 if address space/descriptor storage is exhausted
+ *   (an error dialog will also be raised).
+ *   must be freed via ReleaseAddressSpace.
+**/
+LIB_API void* ReserveAddressSpace(size_t size, size_t commitSize = largePageSize, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE);
+
+/**
+ * release address space and decommit any memory.
+ *
+ * @param p a pointer previously returned by ReserveAddressSpace.
+ * @param size is required by the POSIX implementation and
+ *   ignored on Windows. it also ensures compatibility with UniqueRange.
+ **/
+LIB_API void ReleaseAddressSpace(void* p, size_t size = 0);
+
+
+/**
+ * map physical memory to previously reserved address space.
+ *
+ * @param address, size need not be aligned, but this function commits
+ *   any pages intersecting that interval.
+ * @param pageType, prot - see ReserveAddressSpace.
+ * @return whether memory was successfully committed.
+ *
+ * note: committing only maps virtual pages and does not actually allocate
+ * page frames. Windows XP uses a first-touch heuristic - the page will
+ * be taken from the node whose processor caused the fault.
+ * therefore, worker threads should be the first to write to their memory.
+ *
+ * (this is surprisingly slow in XP, possibly due to PFN lock contention)
+ **/
+LIB_API bool Commit(uintptr_t address, size_t size, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE);
+
+/**
+ * unmap physical memory.
+ *
+ * @return whether the operation succeeded.
+ **/
+LIB_API bool Decommit(uintptr_t address, size_t size);
+
+
+/**
+ * set the memory protection flags for all pages that intersect
+ * the given interval.
+ * the pages must currently be committed.
+ *
+ * @param prot memory protection flags: PROT_NONE or a combination of
+ *   PROT_READ, PROT_WRITE, PROT_EXEC.
+ **/
+LIB_API bool Protect(uintptr_t address, size_t size, int prot);
+
+
+/**
+ * reserve address space and commit memory.
+ *
+ * @param size [bytes] to allocate.
+ * @param pageType, prot - see ReserveAddressSpace.
+ * @return zero-initialized memory aligned to the respective
+ *   page size.
+ **/
+LIB_API void* Allocate(size_t size, PageType pageType = kDefault, int prot = PROT_READ|PROT_WRITE);
+
+/**
+ * decommit memory and release address space.
+ *
+ * @param p a pointer previously returned by Allocate.
+ * @param size is required by the POSIX implementation and
+ *   ignored on Windows. it also ensures compatibility with UniqueRange.
+ *
+ * (this differs from ReleaseAddressSpace, which must account for
+ * extra padding/alignment to largePageSize.)
+ **/
+LIB_API void Free(void* p, size_t size = 0);
+
+
+/**
+ * install a handler that attempts to commit memory whenever a
+ * read/write page fault is encountered. thread-safe.
+ **/
+LIB_API void BeginOnDemandCommits();
+
+/**
+ * decrements the reference count begun by BeginOnDemandCommit and
+ * removes the page fault handler when it reaches 0. thread-safe.
+ **/
+LIB_API void EndOnDemandCommits();
+
+
+LIB_API void DumpStatistics();
+
+}	// namespace vm
+
+#endif	// #ifndef INCLUDED_SYSDEP_VM