diff --git a/source/graphics/Color_asm.asm b/source/graphics/Color_asm.asm index 4cf4344f6f..c094498ca6 100644 --- a/source/graphics/Color_asm.asm +++ b/source/graphics/Color_asm.asm @@ -1,4 +1,4 @@ -%include "../lib/sysdep/ia32.inc" +%include "../lib/sysdep/ia32/ia32.inc" ;------------------------------------------------------------------------------- ; Color conversion (SSE) diff --git a/source/graphics/MapReader.cpp b/source/graphics/MapReader.cpp index 5e37749f85..d4c8d73162 100644 --- a/source/graphics/MapReader.cpp +++ b/source/graphics/MapReader.cpp @@ -224,7 +224,7 @@ int CMapReader::ApplyData() if (unit) { CMatrix3D transform; - memcpy2(&transform._11, m_Objects[i].m_Transform, sizeof(float)*16); + cpu_memcpy(&transform._11, m_Objects[i].m_Transform, sizeof(float)*16); unit->GetModel()->SetTransform(transform); } } diff --git a/source/graphics/Terrain.cpp b/source/graphics/Terrain.cpp index cf67c7cee1..7f99ef2932 100644 --- a/source/graphics/Terrain.cpp +++ b/source/graphics/Terrain.cpp @@ -68,7 +68,7 @@ bool CTerrain::Initialize(u32 size,const u16* data) // given a heightmap? if (data) { // yes; keep a copy of it - memcpy2(m_Heightmap,data,m_MapSize*m_MapSize*sizeof(u16)); + cpu_memcpy(m_Heightmap,data,m_MapSize*m_MapSize*sizeof(u16)); } else { // build a flat terrain memset(m_Heightmap,0,m_MapSize*m_MapSize*sizeof(u16)); @@ -318,7 +318,7 @@ void CTerrain::Resize(u32 size) u16* dst=newHeightmap; u32 copysize=newMapSize>m_MapSize ? m_MapSize : newMapSize; for (j=0;jm_MapSize) { @@ -335,7 +335,7 @@ void CTerrain::Resize(u32 size) src=newHeightmap+((m_MapSize-1)*newMapSize); dst=src+newMapSize; for (u32 i=0;iGetPlayer()->GetColour(); - v.r = i32_from_float(colour.r*255.f); - v.g = i32_from_float(colour.g*255.f); - v.b = i32_from_float(colour.b*255.f); + v.r = cpu_i32_from_float(colour.r*255.f); + v.g = cpu_i32_from_float(colour.g*255.f); + v.b = cpu_i32_from_float(colour.b*255.f); v.a = 255; } else { diff --git a/source/i18n/StrImmutable.h b/source/i18n/StrImmutable.h index 72e500d924..8b42710a83 100644 --- a/source/i18n/StrImmutable.h +++ b/source/i18n/StrImmutable.h @@ -12,6 +12,8 @@ which is not insignificant). #include #include +#include "lib/sysdep/cpu.h" + typedef unsigned short jschar; namespace I18n @@ -35,7 +37,7 @@ namespace I18n ref = new strImW_data; size_t len = wcslen(s)+1; ref->data = new wchar_t[len]; - memcpy2((void*)ref->data, s, len*sizeof(wchar_t)); + cpu_memcpy((void*)ref->data, s, len*sizeof(wchar_t)); } StrImW(const char* s) diff --git a/source/lib/allocators.cpp b/source/lib/allocators.cpp index 07aed0305b..1a10b39cd3 100644 --- a/source/lib/allocators.cpp +++ b/source/lib/allocators.cpp @@ -377,7 +377,7 @@ LibError da_read(DynArray* da, void* data, size_t size) if(da->pos+size > da->cur_size) WARN_RETURN(ERR::FAIL); - memcpy2(data, da->base+da->pos, size); + cpu_memcpy(data, da->base+da->pos, size); da->pos += size; return INFO::OK; } @@ -395,7 +395,7 @@ LibError da_read(DynArray* da, void* data, size_t size) LibError da_append(DynArray* da, const void* data, size_t size) { RETURN_ERR(da_reserve(da, size)); - memcpy2(da->base+da->pos, data, size); + cpu_memcpy(da->base+da->pos, data, size); da->pos += size; return INFO::OK; } diff --git a/source/lib/config.h b/source/lib/config.h index 79ae718eb1..1c52551899 100644 --- a/source/lib/config.h +++ b/source/lib/config.h @@ -319,6 +319,13 @@ # define HAVE_STRDUP 0 #endif +// emulation needed on VC8 because this function is "deprecated" +#define HAVE_MKDIR 1 +#if MSC_VERSION > 800 +# undef HAVE_MKDIR +# define HAVE_MKDIR 0 +#endif + // rint*, fminf, fpclassify (too few/diverse to make separate HAVE_ for each) #define HAVE_C99_MATH 0 #if HAVE_C99 diff --git a/source/lib/debug.cpp b/source/lib/debug.cpp index 4658bf18c1..feeb2615de 100644 --- a/source/lib/debug.cpp +++ b/source/lib/debug.cpp @@ -71,7 +71,7 @@ void debug_wprintf_mem(const wchar_t* fmt, ...) { const size_t copy_size = sizeof(wchar_t) * LOG_CHARS/2; wchar_t* const middle = &debug_log[LOG_CHARS/2]; - memcpy2(debug_log, middle, copy_size); + cpu_memcpy(debug_log, middle, copy_size); memset(middle, 0, copy_size); debug_log_pos -= LOG_CHARS/2; // don't assign middle (may leave gap) } diff --git a/source/lib/lf_alloc.cpp b/source/lib/lf_alloc.cpp index d70131f736..e27edb1f15 100644 --- a/source/lib/lf_alloc.cpp +++ b/source/lib/lf_alloc.cpp @@ -174,7 +174,7 @@ static Descriptor* DescAlloc() { desc = (Descriptor*)AllocNewSB(DESCSBSIZE); // organize descriptors in a linked list - mfence(); + cpu_mfence(); if(CAS(&DescAvail, 0, desc->next)) break; FreeSB((u8*)desc); @@ -190,7 +190,7 @@ static void DescRetire(Descriptor* desc) { old_head = DescAvail; desc->next = old_head; - mfence(); + cpu_mfence(); } while(!CAS(&DescAvail, old_head, desc)); } @@ -385,7 +385,7 @@ static void* MallocFromNewSB(ProcHeap* heap) new_active.credits = MIN(desc->maxcount-1, MAX_CREDITS)-1; desc->anchor.count = (desc->maxcount-1)-(new_active.credits+1); desc->anchor.state = ACTIVE; - mfence(); + cpu_mfence(); if(!CAS(&heap->active, 0, new_active)) { FreeSB(desc->sb); @@ -464,7 +464,7 @@ void lf_free(void* p_) } else new_anchor.count++; - mfence(); + cpu_mfence(); } while(!CAS(&desc->anchor, old_anchor, new_anchor)); if(new_anchor.state == EMPTY) diff --git a/source/lib/lockfree.cpp b/source/lib/lockfree.cpp index 526d41113d..9627b9b2f3 100644 --- a/source/lib/lockfree.cpp +++ b/source/lib/lockfree.cpp @@ -63,11 +63,11 @@ static const size_t MAX_RETIRED = 11; // used to allocate a flat array of all hazard pointers. -// changed via atomic_add by TLS when a thread first calls us / exits. +// changed via cpu_atomic_add by TLS when a thread first calls us / exits. static intptr_t active_threads; // basically module refcount; we can't shut down before it's 0. -// changed via atomic_add by each data structure's init/free. +// changed via cpu_atomic_add by each data structure's init/free. static intptr_t active_data_structures; @@ -142,7 +142,7 @@ static void tls_retire(void* tls_) // successfully marked as unused (must only decrement once) if(CAS(&tls->active, 1, 0)) { - atomic_add(&active_threads, -1); + cpu_atomic_add(&active_threads, -1); debug_assert(active_threads >= 0); } } @@ -223,7 +223,7 @@ static TLS* tls_alloc() have_tls: - atomic_add(&active_threads, 1); + cpu_atomic_add(&active_threads, 1); WARN_ERR(pthread_setspecific(tls_key, tls)); return tls; @@ -459,7 +459,7 @@ LibError lfl_init(LFList* list) } list->head = 0; - atomic_add(&active_data_structures, 1); + cpu_atomic_add(&active_data_structures, 1); return INFO::OK; } @@ -478,7 +478,7 @@ void lfl_free(LFList* list) cur = next; } - atomic_add(&active_data_structures, -1); + cpu_atomic_add(&active_data_structures, -1); debug_assert(active_data_structures >= 0); smr_try_shutdown(); } diff --git a/source/lib/mmgr.cpp b/source/lib/mmgr.cpp index d1519c6d32..526c21aec6 100644 --- a/source/lib/mmgr.cpp +++ b/source/lib/mmgr.cpp @@ -1224,7 +1224,7 @@ void* realloc_dbg(const void* user_p, size_t user_size, AllocType type, const ch // old_size should only be non-zero if the Alloc security checks all passed // If the old buffer was actually zero bytes large, do nothing :P if (old_size && ret) - memcpy2(ret, user_p, MIN(old_size, user_size)); + cpu_memcpy(ret, user_p, MIN(old_size, user_size)); if(user_p) free_dbg(user_p, AT_FREE, file,line,func, stack_frames+1); diff --git a/source/lib/posix/posix.cpp b/source/lib/posix/posix.cpp new file mode 100644 index 0000000000..bbc3ef58e2 --- /dev/null +++ b/source/lib/posix/posix.cpp @@ -0,0 +1,3 @@ +#include "precompiled.h" +#include "posix.h" + diff --git a/source/lib/posix/posix.h b/source/lib/posix/posix.h index 312b49cb99..d4d4941767 100644 --- a/source/lib/posix/posix.h +++ b/source/lib/posix/posix.h @@ -72,4 +72,10 @@ need only be renamed (e.g. _open, _stat). #include "posix_time.h" #include "posix_utsname.h" + +#if OS_MACOSX +extern char* strdup(const char* str); +extern wchar_t* wcsdup(const wchar_t* str); +#endif // #if OS_MACOSX + #endif // #ifndef INCLUDED_POSIX diff --git a/source/lib/posix/posix_filesystem.h b/source/lib/posix/posix_filesystem.h index 022ecb937b..c097bf46f6 100644 --- a/source/lib/posix/posix_filesystem.h +++ b/source/lib/posix/posix_filesystem.h @@ -2,7 +2,12 @@ # include "lib/sysdep/win/wposix/wfilesystem.h" #else # include +# include # include #endif #include "posix_errno.h" // for user convenience + +#if !HAVE_MKDIR +extern int mkdir(const char* path, mode_t mode); +#endif diff --git a/source/lib/res/file/compression.cpp b/source/lib/res/file/compression.cpp index 01d9897e9b..98927f120e 100644 --- a/source/lib/res/file/compression.cpp +++ b/source/lib/res/file/compression.cpp @@ -201,7 +201,7 @@ public: void* cdata_copy = malloc(buf.csize); if(!cdata_copy) WARN_RETURN(ERR::NO_MEM); - memcpy2(cdata_copy, buf.cdata, buf.csize); + cpu_memcpy(cdata_copy, buf.cdata, buf.csize); buf.cdata = (const u8*)cdata_copy; } diff --git a/source/lib/res/file/file.cpp b/source/lib/res/file/file.cpp index 5ef90ff1b1..1527add8ac 100644 --- a/source/lib/res/file/file.cpp +++ b/source/lib/res/file/file.cpp @@ -38,13 +38,6 @@ #include "lib/allocators.h" #include "file_internal.h" -#if OS_WIN -#include // _mkdir -#else -#include -#include -#endif - AT_STARTUP(\ error_setDescription(ERR::FILE_ACCESS, "Insufficient access rights to open file");\ @@ -233,11 +226,7 @@ LibError dir_create(const char* P_path) return INFO::ALREADY_EXISTS; errno = 0; -#if OS_WIN - ret = _mkdir(N_path); -#else - ret = mkdir(N_path, 0777); -#endif + ret = mkdir(N_path, S_IRWXO|S_IRWXU|S_IRWXG); return LibError_from_posix(ret); } diff --git a/source/lib/res/file/file_io.cpp b/source/lib/res/file/file_io.cpp index 586a35996b..2f523c51f2 100644 --- a/source/lib/res/file/file_io.cpp +++ b/source/lib/res/file/file_io.cpp @@ -247,7 +247,7 @@ size_t file_sector_size; // the underlying aio implementation likes buffer and offset to be // sector-aligned; if not, the transfer goes through an align buffer, -// and requires an extra memcpy2. +// and requires an extra cpu_memcpy. // // if the user specifies an unaligned buffer, there's not much we can // do - we can't assume the buffer contains padding. therefore, @@ -513,7 +513,7 @@ class IOManager if(!slot.cached_block && pbuf != FILE_BUF_TEMP && f->flags & FILE_CACHE_BLOCK) { slot.temp_buf = block_cache_alloc(slot.block_id); - memcpy2(slot.temp_buf, block, block_size); + cpu_memcpy(slot.temp_buf, block, block_size); // block_cache_mark_completed will be called in process() } @@ -531,7 +531,7 @@ class IOManager // we have useable data from a previous temp buffer, // but it needs to be copied into the user's buffer if(slot.cached_block && pbuf != FILE_BUF_TEMP) - memcpy2((char*)*pbuf+ofs_misalign+total_transferred, block, block_size); + cpu_memcpy((char*)*pbuf+ofs_misalign+total_transferred, block, block_size); total_transferred += block_size; } diff --git a/source/lib/res/file/path.cpp b/source/lib/res/file/path.cpp index 3a610caeef..c9666717a3 100644 --- a/source/lib/res/file/path.cpp +++ b/source/lib/res/file/path.cpp @@ -262,7 +262,7 @@ const char* file_make_unique_fn_copy(const char* P_fn) DEBUG_WARN_ERR(ERR::NO_MEM); return 0; } - memcpy2((void*)unique_fn, P_fn, fn_len); + cpu_memcpy((void*)unique_fn, P_fn, fn_len); ((char*)unique_fn)[fn_len] = '\0'; atom_map.insert(unique_fn, unique_fn); diff --git a/source/lib/res/file/zip.cpp b/source/lib/res/file/zip.cpp index a9b9b3a76d..fde03f0de6 100644 --- a/source/lib/res/file/zip.cpp +++ b/source/lib/res/file/zip.cpp @@ -277,7 +277,7 @@ static void cdfh_decompose(const CDFH* cdfh_le, // return 0-terminated copy of filename const char* fn_src = (const char*)cdfh_le+CDFH_SIZE; // not 0-terminated! char fn_buf[PATH_MAX]; - memcpy2(fn_buf, fn_src, fn_len*sizeof(char)); + cpu_memcpy(fn_buf, fn_src, fn_len*sizeof(char)); fn_buf[fn_len] = '\0'; fn = file_make_unique_fn_copy(fn_buf); @@ -528,7 +528,7 @@ static LibError lfh_copier_cb(uintptr_t ctx, const void* block, size_t size, siz LFH_Copier* p = (LFH_Copier*)ctx; debug_assert(size <= p->lfh_bytes_remaining); - memcpy2(p->lfh_dst, block, size); + cpu_memcpy(p->lfh_dst, block, size); p->lfh_dst += size; p->lfh_bytes_remaining -= size; @@ -644,7 +644,7 @@ LibError zip_archive_add_file(ZipArchive* za, const ArchiveEntry* ae, void* file WARN_RETURN(ERR::NO_MEM); const size_t slack = za->cdfhs.da.pos-prev_pos - (CDFH_SIZE+fn_len); cdfh_assemble(&p->cdfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->ucsize, fn_len, slack, lfh_ofs); - memcpy2(p->fn, ae->atom_fn, fn_len); + cpu_memcpy(p->fn, ae->atom_fn, fn_len); za->cd_entries++; diff --git a/source/lib/res/graphics/tex.cpp b/source/lib/res/graphics/tex.cpp index ba88ac67ef..e1b72d34ee 100644 --- a/source/lib/res/graphics/tex.cpp +++ b/source/lib/res/graphics/tex.cpp @@ -191,7 +191,7 @@ static void create_level(uint level, uint level_w, uint level_h, if(level == 0) { debug_assert(level_data_size == cld->prev_level_data_size); - memcpy2(dst, src, level_data_size); + cpu_memcpy(dst, src, level_data_size); } else { @@ -317,7 +317,7 @@ TIMER_ACCRUE(tc_plain_transform); void* new_data = mem_alloc(data_size, 4*KiB, 0, &hm); if(!new_data) WARN_RETURN(ERR::NO_MEM); - memcpy2(new_data, data, data_size); + cpu_memcpy(new_data, data, data_size); // setup row source/destination pointers (simplifies outer loop) u8* dst = (u8*)new_data; @@ -338,7 +338,7 @@ TIMER_ACCRUE(tc_plain_transform); { for(uint y = 0; y < h; y++) { - memcpy2(dst, src, pitch); + cpu_memcpy(dst, src, pitch); dst += pitch; src += row_ofs; } diff --git a/source/lib/res/h_mgr.cpp b/source/lib/res/h_mgr.cpp index 924491b804..453c6a1f7f 100644 --- a/source/lib/res/h_mgr.cpp +++ b/source/lib/res/h_mgr.cpp @@ -451,7 +451,7 @@ static void fn_store(HDATA* hd, const char* fn) WARN_ERR_RETURN(ERR::NO_MEM); } - memcpy2((void*)hd->fn, fn, size); // faster than strcpy + cpu_memcpy((void*)hd->fn, fn, size); // faster than strcpy } // TODO: store this in a flag - faster. diff --git a/source/lib/res/sound/ogghack.cpp b/source/lib/res/sound/ogghack.cpp index bb84b89f89..63d1b09e63 100644 --- a/source/lib/res/sound/ogghack.cpp +++ b/source/lib/res/sound/ogghack.cpp @@ -10,6 +10,7 @@ #include #include "ogghack.h" // HACK: must be included after al.h (bad interface) +#include "lib/sysdep/cpu.h" #if MSC_VERSION # ifdef NDEBUG @@ -62,7 +63,7 @@ size_t read_func(void* ptr, size_t elements, size_t el_size, void* datasource) Buf& b = incoming_bufs->front(); size_t copy_size = std::min(b.left, size); - memcpy2(ptr, (char*)b.p+b.pos, copy_size); + cpu_memcpy(ptr, (char*)b.p+b.pos, copy_size); total_read += copy_size; b.pos += copy_size; b.left -= copy_size; @@ -108,7 +109,7 @@ void ogg_give_raw(void* _o, void* p, size_t size) IncomingBufs* incoming_bufs = &o->incoming_bufs; void* copy = malloc(size); - memcpy2(copy, p, size); + cpu_memcpy(copy, p, size); incoming_bufs->push_back(Buf(copy, size)); } diff --git a/source/lib/sysdep/cpu.cpp b/source/lib/sysdep/cpu.cpp index 9efe1b07e8..10bcba6e05 100644 --- a/source/lib/sysdep/cpu.cpp +++ b/source/lib/sysdep/cpu.cpp @@ -24,102 +24,275 @@ #include "cpu.h" #include "lib/lib.h" +#include "lib/posix/posix.h" #if CPU_IA32 -# include "lib/sysdep/ia32.h" +# include "lib/sysdep/ia32/ia32.h" +# include "lib/sysdep/ia32/ia32_memcpy.h" +#endif +#if OS_BSD +# include "lib/sysdep/unix/bsd.h" +#endif +#if OS_WIN +# include "lib/sysdep/win/wcpu.h" +# include "lib/sysdep/win/wposix/wtime_internal.h" // HACK (see call to wtime_reset_impl) #endif AT_STARTUP(\ error_setDescription(ERR::CPU_FEATURE_MISSING, "This CPU doesn't support a required feature");\ error_setDescription(ERR::CPU_UNKNOWN_OPCODE, "Disassembly failed");\ + error_setDescription(ERR::CPU_UNKNOWN_VENDOR, "CPU vendor unknown");\ error_setDescription(ERR::CPU_RESTRICTED_AFFINITY, "Cannot set desired CPU affinity");\ ) -char cpu_type[CPU_TYPE_LEN] = ""; -double cpu_freq = 0.f; - -// -1 if detect not yet called, or cannot be determined -int cpus = -1; -int cpu_ht_units = -1; -int cpu_cores = -1; -int cpu_speedstep = -1; +static ModuleInitState module_init_state = MODULE_BEFORE_INIT; -void cpu_init() +//----------------------------------------------------------------------------- +#pragma region Accessor functions +// prevent other modules from changing the underlying data. + +bool cpu_isModuleInitialized() +{ + return module_init_state == MODULE_INITIALIZED; +} + +const char* cpu_identifierString() { #if CPU_IA32 - // must come before any uses of ia32.asm, e.g. by get_cpu_info - ia32_init(); + return ia32_identifierString(); +#endif +} + +double cpu_clockFrequency() +{ +#if CPU_IA32 + return ia32_clockFrequency(); // authoritative, precise +#endif +} + +int cpu_numPackages() +{ +#if CPU_IA32 + return ia32_numPackages(); +#endif +} + +int cpu_coresPerPackage() +{ +#if CPU_IA32 + return ia32_coresPerPackage(); +#endif +} + +int cpu_logicalPerPackage() +{ +#if CPU_IA32 + return ia32_logicalPerPackage(); +#endif +} + + +bool cpu_isThrottlingPossible() +{ +#if CPU_IA32 + if(ia32_isThrottlingPossible() == 1) + return true; +#endif +#if OS_WIN + if(wcpu_isThrottlingPossible() == 1) + return true; +#endif + return false; +} + +#pragma endregion +//----------------------------------------------------------------------------- +// memory + +static size_t cpu_page_size = 0; +// determined during cpu_init; cleaned up and given in MiB +static size_t cpu_memory_total_mib = 0; + +// System V derived (GNU/Linux, Solaris) +#if defined(_SC_AVPHYS_PAGES) + +static int sysconfFromMemType(CpuMemoryIndicators mem_type) +{ + switch(mem_type) + { + case CPU_MEM_TOTAL: + return _SC_PHYS_PAGES; + case CPU_MEM_AVAILABLE: + return _SC_AVPHYS_PAGES; + } + UNREACHABLE; +} + +#endif + +size_t cpu_memorySize(CpuMemoryIndicators mem_type) +{ + // quasi-POSIX +#if defined(_SC_AVPHYS_PAGES) + const int sc_name = sysconfFromMemType(mem_type); + const size_t memory_size = sysconf(sc_name) * cpu_page_size; + return memory_size; + // BSD / Mac OS X +#else + return bsd_memorySize(mem_type); +#endif +} + + +static size_t calcMemoryTotalMiB() +{ + size_t memory_total = cpu_memorySize(CPU_MEM_TOTAL); + const size_t memory_total_pow2 = (size_t)round_up_to_pow2((uint)memory_total); + // .. difference too great, just round up to 1 MiB + if(memory_total_pow2 - memory_total > 3*MiB) + memory_total = round_up(memory_total, 1*MiB); + // .. difference acceptable, use next power of two + else + memory_total = memory_total_pow2; + const size_t memory_total_mib = memory_total / MiB; + return memory_total_mib; +} + +size_t cpu_memoryTotalMiB() +{ + return cpu_memory_total_mib; +} + + +//----------------------------------------------------------------------------- + +#if CPU_IA32 + +static void initAndConfigureIA32() +{ + ia32_init(); // must come before any use of ia32* + + ia32_memcpy_init(); // no longer set 24 bit (float) precision by default: for // very long game uptimes (> 1 day; e.g. dedicated server), // we need full precision when calculating the time. // if there's a spot where we want to speed up divides|sqrts, // we can temporarily change precision there. - //ia32_control87(IA32_PC_24, IA32_MCW_PC); + //ia32_asm_control87(IA32_PC_24, IA32_MCW_PC); // to help catch bugs, enable as many floating-point exceptions as - // possible. that means only zero-divide, because the JS engine is - // triggering the rest. + // possible. unfortunately SpiderMonkey triggers all of them. // note: passing a flag *disables* that exception. - ia32_control87(IA32_EM_ZERODIVIDE|IA32_EM_INVALID|IA32_EM_DENORMAL|IA32_EM_OVERFLOW|IA32_EM_UNDERFLOW|IA32_EM_INEXACT, IA32_MCW_EM); + ia32_asm_control87(IA32_EM_ZERODIVIDE|IA32_EM_INVALID|IA32_EM_DENORMAL|IA32_EM_OVERFLOW|IA32_EM_UNDERFLOW|IA32_EM_INEXACT, IA32_MCW_EM); // no longer round toward zero (truncate). changing this setting // resulted in much faster float->int casts, because the compiler // could be told (via /QIfist) to use FISTP while still truncating // the result as required by ANSI C. however, FPU calculation // results were changed significantly, so it had to be disabled. - //ia32_control87(IA32_RC_CHOP, IA32_MCW_RC); -#endif - - // detects CPU clock frequency and capabilities, which are prerequisites - // for using the TSC as a timer (desirable due to its high resolution). - // do this before lengthy init so we can time those accurately. -#if OS_WIN - extern LibError win_get_cpu_info(); - win_get_cpu_info(); -#elif OS_UNIX - extern LibError unix_get_cpu_info(); - unix_get_cpu_info(); -#endif - -#if CPU_IA32 - ia32_get_cpu_info(); -#endif + //ia32_asm_control87(IA32_RC_CHOP, IA32_MCW_RC); } +#endif -// -// memory -// - -size_t tot_mem = 0; -size_t avl_mem = 0; - -size_t page_size = 0; - -void get_mem_status() +void cpu_init() { -#ifdef _SC_PAGESIZE - if(!page_size) - page_size = (size_t)sysconf(_SC_PAGESIZE); +#if CPU_IA32 + initAndConfigureIA32(); #endif - // Sys V derived (GNU/Linux, Solaris) -#if defined(_SC_AVPHYS_PAGES) + // memory + cpu_page_size = (size_t)sysconf(_SC_PAGESIZE); + cpu_memory_total_mib = calcMemoryTotalMiB(); - tot_mem = sysconf(_SC_PHYS_PAGES ) * page_size; - avl_mem = sysconf(_SC_AVPHYS_PAGES) * page_size; - - // BSD / Mac OS X -#elif defined(HW_PHYSMEM) - - size_t len = sizeof(tot_mem); - int mib[2] = { CTL_HW, HW_PHYSMEM }; - sysctl(mib, 2, &tot_mem, &len, 0, 0); - mib[1] = HW_USERMEM; - sysctl(mib, 2, &avl_mem, &len, 0, 0); + // must be set before wtime_reset_impl since it queries this flag via + // cpu_isModuleInitialized. + module_init_state = MODULE_INITIALIZED; + // HACK: on Windows, the HRT makes its final implementation choice + // in the first calibrate call where cpu info is available. + // call wtime_reset_impl here to have that happen now so app code isn't + // surprised by a timer change, although the HRT does try to + // keep the timer continuous. +#if OS_WIN + wtime_reset_impl(); +#endif +} + + +//----------------------------------------------------------------------------- + +bool cpu_CAS(uintptr_t* location, uintptr_t expected, uintptr_t new_value) +{ +#if CPU_IA32 + return ia32_asm_CAS(location, expected, new_value); +#endif +} + +void cpu_atomic_add(intptr_t* location, intptr_t increment) +{ +#if CPU_IA32 + return ia32_asm_atomic_add(location, increment); +#endif +} + +void cpu_mfence() +{ +#if CPU_IA32 + return ia32_mfence(); +#endif +} + +void cpu_serialize() +{ +#if CPU_IA32 + return ia32_serialize(); +#endif +} + +void* cpu_memcpy(void* RESTRICT dst, const void* RESTRICT src, size_t nbytes) +{ +#if CPU_IA32 + return ia32_memcpy(dst, src, nbytes); +#else + return memcpy(dst, src, nbytes); +#endif +} + +LibError cpu_callByEachCPU(CpuCallback cb, void* param) +{ +#if OS_WIN + return wcpu_callByEachCPU(cb, param); +#endif +} + + +i32 cpu_i32_from_float(float f) +{ +#if USE_IA32_FLOAT_TO_INT + return ia32_asm_i32_from_float(f); +#else + return (i32)f; +#endif +} + +i32 cpu_i32_from_double(double d) +{ +#if USE_IA32_FLOAT_TO_INT + return ia32_asm_i32_from_double(d); +#else + return (i32)d; +#endif +} + +i64 cpu_i64_from_double(double d) +{ +#if USE_IA32_FLOAT_TO_INT + return ia32_asm_i64_from_double(d); +#else + return (i64)d; #endif } diff --git a/source/lib/sysdep/cpu.h b/source/lib/sysdep/cpu.h index 224257f5ef..c7e923fb11 100644 --- a/source/lib/sysdep/cpu.h +++ b/source/lib/sysdep/cpu.h @@ -23,57 +23,99 @@ #ifndef CPU_H__ #define CPU_H__ +#ifdef __cplusplus +extern "C" { +#endif namespace ERR { const LibError CPU_FEATURE_MISSING = -130000; const LibError CPU_UNKNOWN_OPCODE = -130001; - const LibError CPU_RESTRICTED_AFFINITY = -130002; + const LibError CPU_UNKNOWN_VENDOR = -130002; + const LibError CPU_RESTRICTED_AFFINITY = -130003; } -const size_t CPU_TYPE_LEN = 49; // IA32 processor brand string is <= 48 chars -extern char cpu_type[CPU_TYPE_LEN]; - -extern double cpu_freq; - - -// -1 if detect not yet called, or cannot be determined: - -extern "C" int cpus; // # packages (i.e. sockets; > 1 => SMP system) -extern int cpu_ht_units; // degree of hyperthreading, typically 2 -extern int cpu_cores; // cores per package, typically 2 - -extern int cpu_speedstep; - - +// must be called before any of the below accessors. extern void cpu_init(void); +extern bool cpu_isModuleInitialized(); -extern size_t tot_mem; -extern size_t avl_mem; -// updates *_mem above -extern void get_mem_status(void); +extern const char* cpu_identifierString(); +extern double cpu_clockFrequency(); +extern int cpu_numPackages(); // i.e. sockets +extern int cpu_coresPerPackage(); +extern int cpu_logicalPerPackage(); +extern bool cpu_isThrottlingPossible(); +// +// memory +// + +enum CpuMemoryIndicators +{ + CPU_MEM_TOTAL, CPU_MEM_AVAILABLE +}; + +extern size_t cpu_memorySize(CpuMemoryIndicators mem_type); + +// faster than cpu_memorySize (caches total size determined during init), +// returns #Mebibytes (cleaned up to account e.g. for nonpaged pool) +extern size_t cpu_memoryTotalMiB(); + + +// +// misc +// + // atomic "compare and swap". compare the machine word at against // ; if not equal, return false; otherwise, overwrite it with // and return true. -extern "C" bool CAS_(uintptr_t* location, uintptr_t expected, uintptr_t new_value); +extern bool cpu_CAS(uintptr_t* location, uintptr_t expected, uintptr_t new_value); // this is often used for pointers, so the macro coerces parameters to // uinptr_t. invalid usage unfortunately also goes through without warnings. // to catch cases where the caller has passed as or // similar mishaps, the implementation verifies is a valid pointer. -#define CAS(l,o,n) CAS_((uintptr_t*)l, (uintptr_t)o, (uintptr_t)n) +#define CAS(l,o,n) cpu_CAS((uintptr_t*)l, (uintptr_t)o, (uintptr_t)n) -extern "C" void atomic_add(intptr_t* location, intptr_t increment); +extern void cpu_atomic_add(intptr_t* location, intptr_t increment); // enforce strong memory ordering. -extern "C" void mfence(); +extern void cpu_mfence(); + +extern void cpu_serialize(); + + +// drop-in replacement for libc memcpy(). only requires CPU support for +// MMX (by now universal). highly optimized for Athlon and Pentium III +// microarchitectures; significantly outperforms VC7.1 memcpy and memcpy_amd. +// for details, see accompanying article. +extern void* cpu_memcpy(void* RESTRICT dst, const void* RESTRICT src, size_t size); + + +// execute the specified function once on each CPU. +// this includes logical HT units and proceeds serially (function +// is never re-entered) in order of increasing OS CPU ID. +// note: implemented by switching thread affinity masks and forcing +// a reschedule, which is apparently not possible with POSIX. +// +// may fail if e.g. OS is preventing us from running on some CPUs. +// called from ia32.cpp get_cpu_count. + +typedef void (*CpuCallback)(void* param); +extern LibError cpu_callByEachCPU(CpuCallback cb, void* param); + + +// convert float to int much faster than _ftol2, which would normally be +// used by (int) casts. +extern i32 cpu_i32_from_float(float f); +extern i32 cpu_i32_from_double(double d); +extern i64 cpu_i64_from_double(double d); + -extern "C" void serialize(); // Win32 CONTEXT field abstraction @@ -88,4 +130,8 @@ extern "C" void serialize(); # define SP_ Esp #endif +#ifdef __cplusplus +} +#endif + #endif // #ifndef CPU_H__ diff --git a/source/lib/sysdep/ia32.cpp b/source/lib/sysdep/ia32/ia32.cpp similarity index 57% rename from source/lib/sysdep/ia32.cpp rename to source/lib/sysdep/ia32/ia32.cpp index 9622dabb83..844930e67e 100644 --- a/source/lib/sysdep/ia32.cpp +++ b/source/lib/sysdep/ia32/ia32.cpp @@ -31,147 +31,81 @@ #include "lib/posix/posix_pthread.h" #include "lib/lib.h" #include "lib/timer.h" -#include "cpu.h" - -// HACK (see call to wtime_reset_impl) -#if OS_WIN -#include "lib/sysdep/win/wposix/wtime_internal.h" -#endif +#include "lib/sysdep/cpu.h" #if !HAVE_MS_ASM && !HAVE_GNU_ASM #error ia32.cpp needs inline assembly support! #endif -extern "C" { - -// set by ia32_init, referenced by ia32_memcpy (asm) -extern u32 ia32_memcpy_size_mask = 0; - -void ia32_init() -{ - ia32_asm_init(); - - // memcpy init: set the mask that is applied to transfer size before - // choosing copy technique. this is the mechanism for disabling - // codepaths that aren't supported on all CPUs; see article for details. - // .. check for PREFETCHNTA and MOVNTQ support. these are part of the SSE - // instruction set, but also supported on older Athlons as part of - // the extended AMD MMX set. - if(ia32_cap(IA32_CAP_SSE) || ia32_cap(IA32_CAP_AMD_MMX_EXT)) - ia32_memcpy_size_mask = ~0u; -} - - -//----------------------------------------------------------------------------- -// fast implementations of some sysdep.h functions; see documentation there //----------------------------------------------------------------------------- +// capability bits -#if HAVE_MS_ASM +// set by ia32_cap_init, referenced by ia32_cap +// treated as 128 bit field; order: std ecx, std edx, ext ecx, ext edx +// keep in sync with enum CpuCap! +static u32 ia32_caps[4]; -// notes: -// - declspec naked is significantly faster: it avoids redundant -// store/load, even though it prevents inlining. -// - stupid VC7 gets arguments wrong when using __declspec(naked); -// we need to use DWORD PTR and esp-relative addressing. -// if on 64-bit systems, [esp+4] will have to change -cassert(sizeof(int)*CHAR_BIT == 32); - -__declspec(naked) float ia32_rintf(float) +static void ia32_cap_init() { - __asm fld [esp+4] - __asm frndint - __asm ret -} - -__declspec(naked) double ia32_rint(double) -{ - __asm fld QWORD PTR [esp+4] - __asm frndint - __asm ret -} - -__declspec(naked) float ia32_fminf(float, float) -{ - __asm + u32 regs[4]; + if(ia32_asm_cpuid(1, regs)) { - fld DWORD PTR [esp+4] - fld DWORD PTR [esp+8] - fcomi st(0), st(1) - fcmovnb st(0), st(1) - fxch - fstp st(0) - ret + ia32_caps[0] = regs[ECX]; + ia32_caps[1] = regs[EDX]; + } + if(ia32_asm_cpuid(0x80000001, regs)) + { + ia32_caps[2] = regs[ECX]; + ia32_caps[3] = regs[EDX]; } } -__declspec(naked) float ia32_fmaxf(float, float) +bool ia32_cap(IA32Cap cap) { - __asm + const uint tbl_idx = cap >> 5; + const uint bit_idx = cap & 0x1f; + if(tbl_idx > 3) { - fld DWORD PTR [esp+4] - fld DWORD PTR [esp+8] - fcomi st(0), st(1) - fcmovb st(0), st(1) - fxch - fstp st(0) - ret + DEBUG_WARN_ERR(ERR::INVALID_PARAM); + return false; } + return (ia32_caps[tbl_idx] & BIT(bit_idx)) != 0; } -#endif // HAVE_MS_ASM - -#if USE_IA32_FLOAT_TO_INT // implies HAVE_MS_ASM - -// notes: -// - PTR is necessary because __declspec(naked) means the assembler -// cannot refer to parameter argument type to get it right. -// - to conform with the fallback implementation (a C cast), we need to -// end up with truncate/"chop" rounding. subtracting does the trick, -// assuming RC is the IA-32 default round-to-nearest mode. - -static const float round_bias = 0.4999999f; - -__declspec(naked) i32 ia32_i32_from_float(float f) +// we only store enum Vendor rather than the string because that +// is easier to compare. +static enum Vendor { - UNUSED2(f); -__asm{ - push eax - fld DWORD PTR [esp+8] - fsub [round_bias] - fistp DWORD PTR [esp] - pop eax - ret -}} + UNKNOWN, INTEL, AMD +} +vendor = UNKNOWN; -__declspec(naked) i32 ia32_i32_from_double(double d) +static void detectVendor() { - UNUSED2(d); -__asm{ - push eax - fld QWORD PTR [esp+8] - fsub [round_bias] - fistp DWORD PTR [esp] - pop eax - ret -}} + u32 regs[4]; + if(!ia32_asm_cpuid(0, regs)) + return; + + // copy regs to string + // note: 'strange' ebx,edx,ecx reg order is due to ModR/M encoding order. + char vendor_str[13]; + u32* vendor_str_u32 = (u32*)vendor_str; + vendor_str_u32[0] = regs[EBX]; + vendor_str_u32[1] = regs[EDX]; + vendor_str_u32[2] = regs[ECX]; + vendor_str[12] = '\0'; // 0-terminate + + if(!strcmp(vendor_str, "AuthenticAMD")) + vendor = AMD; + else if(!strcmp(vendor_str, "GenuineIntel")) + vendor = INTEL; + else + DEBUG_WARN_ERR(ERR::CPU_UNKNOWN_VENDOR); +} -__declspec(naked) i64 ia32_i64_from_double(double d) -{ - UNUSED2(d); -__asm{ - push edx - push eax - fld QWORD PTR [esp+12] - fsub [round_bias] - fistp QWORD PTR [esp] - pop eax - pop edx - ret -}} -#endif // USE_IA32_FLOAT_TO_INT //----------------------------------------------------------------------------- @@ -179,7 +113,7 @@ __asm{ // this RDTSC implementation writes edx:eax to a temporary and returns that. // rationale: this insulates against changing compiler calling conventions, // at the cost of some efficiency. -// use ia32_rdtsc_edx_eax instead if the return convention is known to be +// use ia32_asm_rdtsc_edx_eax instead if the return convention is known to be // edx:eax (should be the case on all 32-bit x86). u64 ia32_rdtsc_safe() { @@ -213,7 +147,7 @@ void ia32_debug_break() // (SIGTRAP) is most probably available if HAVE_GNU_ASM. // we include it for completeness, though. #elif HAVE_GNU_ASM - __asm__ __volatile__ ("mfence"); + __asm__ __volatile__ ("int $3"); #endif } @@ -223,7 +157,7 @@ void ia32_debug_break() //----------------------------------------------------------------------------- // enforce strong memory ordering. -void mfence() +void ia32_mfence() { // Pentium IV if(ia32_cap(IA32_CAP_SSE2)) @@ -234,7 +168,7 @@ void mfence() #endif } -void serialize() +void ia32_serialize() { #if HAVE_MS_ASM __asm cpuid @@ -248,100 +182,37 @@ void serialize() // CPU / feature detect //----------------------------------------------------------------------------- -bool ia32_cap(IA32Cap cap) +// returned in edx by CPUID 0x80000007. +enum AmdPowerNowFlags { - // treated as 128 bit field; order: std ecx, std edx, ext ecx, ext edx - // keep in sync with enum CpuCap! - static u32 caps[4]; - ONCE(\ - u32 regs[4]; - if(ia32_cpuid(1, regs))\ - {\ - caps[0] = regs[ECX];\ - caps[1] = regs[EDX];\ - }\ - if(ia32_cpuid(0x80000001, regs))\ - {\ - caps[2] = regs[ECX];\ - caps[3] = regs[EDX];\ - }\ - ); - - const uint tbl_idx = cap >> 5; - const uint bit_idx = cap & 0x1f; - if(tbl_idx > 3) - { - debug_warn("cap invalid"); - return false; - } - return (caps[tbl_idx] & BIT(bit_idx)) != 0; -} - - - - - -// we only store enum Vendor rather than the string because that -// is easier to compare. -enum Vendor { UNKNOWN, INTEL, AMD }; -static Vendor vendor = UNKNOWN; - - - -enum MiscCpuCapBits -{ - // AMD PowerNow! flags (returned in edx by CPUID 0x80000007) POWERNOW_FREQ_ID_CTRL = 2 }; - -static void get_cpu_vendor() +const char* ia32_identifierString() { - u32 regs[4]; - if(!ia32_cpuid(0, regs)) - return; + // 3 calls x 4 registers x 4 bytes = 48 + static char identifier_string[48+1] = ""; - // copy regs to string - // note: 'strange' ebx,edx,ecx reg order is due to ModR/M encoding order. - char vendor_str[13]; - u32* vendor_str_u32 = (u32*)vendor_str; - vendor_str_u32[0] = regs[EBX]; - vendor_str_u32[1] = regs[EDX]; - vendor_str_u32[2] = regs[ECX]; - vendor_str[12] = '\0'; // 0-terminate - - if(!strcmp(vendor_str, "AuthenticAMD")) - vendor = AMD; - else if(!strcmp(vendor_str, "GenuineIntel")) - vendor = INTEL; - else - debug_warn("unknown vendor"); -} - - -static void get_cpu_type() -{ // get processor signature u32 regs[4]; - if(!ia32_cpuid(1, regs)) - debug_warn("cpuid 1 failed"); + if(!ia32_asm_cpuid(1, regs)) + DEBUG_WARN_ERR(ERR::CPU_FEATURE_MISSING); const uint model = bits(regs[EAX], 4, 7); const uint family = bits(regs[EAX], 8, 11); // get brand string (if available) - // note: ia32_cpuid writes 4 u32s directly to cpu_type - + // note: ia32_asm_cpuid writes 4 u32s directly to identifier_string - // be very careful with pointer arithmetic! - u32* cpu_type_u32 = (u32*)cpu_type; + u32* u32_string = (u32*)identifier_string; bool have_brand_string = false; - if(ia32_cpuid(0x80000002, cpu_type_u32+0 ) && - ia32_cpuid(0x80000003, cpu_type_u32+4) && - ia32_cpuid(0x80000004, cpu_type_u32+8)) + if(ia32_asm_cpuid(0x80000002, u32_string+0 ) && + ia32_asm_cpuid(0x80000003, u32_string+4) && + ia32_asm_cpuid(0x80000004, u32_string+8)) have_brand_string = true; - - // note: cpu_type is guaranteed to hold 48+1 chars, since that's the - // length of the CPU brand string => we can safely copy short literals. + // note: we previously verified max_chars is long enough, so copying + // short literals into it is safe. // fall back to manual detect of CPU type because either: // - CPU doesn't support brand string (we use a flag to indicate this @@ -349,7 +220,7 @@ static void get_cpu_type() // - the brand string is useless, e.g. "Unknown". this happens on // some older boards whose BIOS reprograms the string for CPUs it // doesn't recognize. - if(!have_brand_string || strncmp(cpu_type, "Unknow", 6) == 0) + if(!have_brand_string || strncmp(identifier_string, "Unknow", 6) == 0) { if(vendor == AMD) { @@ -357,15 +228,15 @@ static void get_cpu_type() if(family == 6) { if(model == 3 || model == 7) - SAFE_STRCPY(cpu_type, "AMD Duron"); + SAFE_STRCPY(identifier_string, "AMD Duron"); else if(model <= 5) - SAFE_STRCPY(cpu_type, "AMD Athlon"); + SAFE_STRCPY(identifier_string, "AMD Athlon"); else { if(ia32_cap(IA32_CAP_AMD_MP)) - SAFE_STRCPY(cpu_type, "AMD Athlon MP"); + SAFE_STRCPY(identifier_string, "AMD Athlon MP"); else - SAFE_STRCPY(cpu_type, "AMD Athlon XP"); + SAFE_STRCPY(identifier_string, "AMD Athlon XP"); } } } @@ -375,139 +246,60 @@ static void get_cpu_type() if(family == 6) { if(model == 1) - SAFE_STRCPY(cpu_type, "Intel Pentium Pro"); + SAFE_STRCPY(identifier_string, "Intel Pentium Pro"); else if(model == 3 || model == 5) - SAFE_STRCPY(cpu_type, "Intel Pentium II"); + SAFE_STRCPY(identifier_string, "Intel Pentium II"); else if(model == 6) - SAFE_STRCPY(cpu_type, "Intel Celeron"); + SAFE_STRCPY(identifier_string, "Intel Celeron"); else - SAFE_STRCPY(cpu_type, "Intel Pentium III"); + SAFE_STRCPY(identifier_string, "Intel Pentium III"); } } } - // cpu_type already holds a valid brand string; pretty it up. + // identifier_string already holds a valid brand string; pretty it up. else { // strip (tm) from Athlon string - if(!strncmp(cpu_type, "AMD Athlon(tm)", 14)) - memmove(cpu_type+10, cpu_type+14, 35); + if(!strncmp(identifier_string, "AMD Athlon(tm)", 14)) + memmove(identifier_string+10, identifier_string+14, 35); // remove 2x (R) and CPU freq from P4 string float freq; // we can't use this because it isn't necessarily correct - the CPU // may be overclocked. a variable must be passed, though, since // scanf returns the number of fields actually stored. - if(sscanf(cpu_type, " Intel(R) Pentium(R) 4 CPU %fGHz", &freq) == 1) - SAFE_STRCPY(cpu_type, "Intel Pentium 4"); + if(sscanf(identifier_string, " Intel(R) Pentium(R) 4 CPU %fGHz", &freq) == 1) + SAFE_STRCPY(identifier_string, "Intel Pentium 4"); } + + return identifier_string; } -//----------------------------------------------------------------------------- - -static uint log_id_bits; // bit index; divides APIC ID into log and phys - -static const uint INVALID_ID = ~0u; -static uint last_phys_id = INVALID_ID, last_log_id = INVALID_ID; -static uint phys_ids = 0, log_ids = 0; - -// count # distinct physical and logical APIC IDs for get_cpu_count. -// called on each OS-visible "CPU" by on_each_cpu. -static void count_ids() -{ - // get APIC id - u32 regs[4]; - if(!ia32_cpuid(1, regs)) - debug_warn("cpuid 1 failed"); - const uint id = bits(regs[EBX], 24, 31); - - // partition into physical and logical ID - const uint phys_id = bits(id, 0, log_id_bits-1); - const uint log_id = bits(id, log_id_bits, 7); - - // note: APIC IDs are assigned sequentially, so we compare against the - // last one encountered. - if(last_phys_id != INVALID_ID && last_phys_id != phys_id) - cpus++; - if(last_log_id != INVALID_ID && last_log_id != log_id ) - cpus++; - last_phys_id = phys_id; - last_log_id = log_id; -} - - -// fix CPU count reported by OS (incorrect if HT active or multicore); -// also separates it into cpu_ht_units and cpu_cores. -static void get_cpu_count() -{ - debug_assert(cpus > 0 && "must know # 'CPU's (call OS-specific detect first)"); - - // get # "logical CPUs" per package (uniform over all packages). - // TFM is unclear but seems to imply this includes HT units *and* cores! - u32 regs[4]; - if(!ia32_cpuid(1, regs)) - debug_warn("ia32_cpuid(1) failed"); - const uint log_cpu_per_package = bits(regs[EBX], 16, 23); - // .. and # cores - if(ia32_cpuid(4, regs)) - cpu_cores = bits(regs[EBX], 26, 31)+1; - else - cpu_cores = 1; - - // if HT is active (enabled in BIOS and OS), we have a problem: - // OSes (Windows at least) report # CPUs as packages * cores * HT_units. - // there is no direct way to determine if HT is actually enabled, - // so if it is supported, we have to examine all APIC IDs and - // figure out what kind of "CPU" each one is. *sigh* - // - // note: we don't check if it's Intel and P4 or above - HT may be - // supported on other CPUs in future. all processors should set this - // feature bit correctly, so it's not a problem. - if(ia32_cap(IA32_CAP_HT)) - { - log_id_bits = log2(log_cpu_per_package); // see above - last_phys_id = last_log_id = INVALID_ID; - phys_ids = log_ids = 0; - if(sys_on_each_cpu(count_ids) == 0) - { - cpus = phys_ids; - cpu_ht_units = log_ids / cpu_cores; - return; // this is authoritative - } - // OS apparently doesn't support CPU affinity. - // HT might be disabled, but return # units anyway. - else - cpu_ht_units = log_cpu_per_package / cpu_cores; - } - // not HT-capable; return 1 to allow total = cpus * HT_units * cores. - else - cpu_ht_units = 1; - - cpus /= cpu_cores; -} - - - - -static void check_for_speedstep() +int ia32_isThrottlingPossible() { if(vendor == INTEL) { if(ia32_cap(IA32_CAP_EST)) - cpu_speedstep = 1; + return 1; } else if(vendor == AMD) { u32 regs[4]; - if(ia32_cpuid(0x80000007, regs)) + if(ia32_asm_cpuid(0x80000007, regs)) + { if(regs[EDX] & POWERNOW_FREQ_ID_CTRL) - cpu_speedstep = 1; + return 1; + } } + + return 0; // pretty much authoritative, so don't return -1. } - -static void measure_cpu_freq() +double ia32_clockFrequency() { + double clock_frequency = 0.0; + // set max priority, to reduce interference while measuring. int old_policy; static sched_param old_param; // (static => 0-init) pthread_getschedparam(pthread_self(), &old_policy, &old_param); @@ -582,35 +374,141 @@ static void measure_cpu_freq() const int lo = num_samples/4, hi = 3*num_samples/4; for(i = lo; i < hi; i++) sum += samples[i]; - cpu_freq = sum / (hi-lo); + clock_frequency = sum / (hi-lo); } // else: TSC not available, can't measure; cpu_freq remains unchanged. // restore previous policy and priority. pthread_setschedparam(pthread_self(), old_policy, &old_param); + + return clock_frequency; } -void ia32_get_cpu_info() +// (these are uniform over all packages) + + +static uint logicalPerPackage() { - get_cpu_vendor(); - get_cpu_type(); - get_cpu_count(); - check_for_speedstep(); - measure_cpu_freq(); + // not Hyperthreading capable + if(!ia32_cap(IA32_CAP_HT)) + return 1; - // HACK: on Windows, the HRT makes its final implementation choice - // in the first calibrate call where cpu info is available. - // call wtime_reset_impl here to have that happen now, - // so app code isn't surprised by a timer change, although the HRT - // does try to keep the timer continuous. -#if OS_WIN - wtime_reset_impl(); -#endif + u32 regs[4]; + if(!ia32_asm_cpuid(1, regs)) + DEBUG_WARN_ERR(ERR::CPU_FEATURE_MISSING); + const uint logical_per_package = bits(regs[EBX], 16, 23); + return logical_per_package; } +static uint coresPerPackage() +{ + // single core + u32 regs[4]; + if(!ia32_asm_cpuid(4, regs)) + return 1; + + const uint cores_per_package = bits(regs[EAX], 26, 31)+1; + return cores_per_package; +} + + +typedef std::vector Ids; +typedef std::set IdSet; + +static void storeApicId(void* param) +{ + u32 regs[4]; + if(!ia32_asm_cpuid(1, regs)) + DEBUG_WARN_ERR(ERR::CPU_FEATURE_MISSING); + const uint apic_id = bits(regs[EBX], 24, 31); + + Ids* apic_ids = (Ids*)param; + apic_ids->push_back(apic_id); +} + +static void extractFieldsIntoSet(const Ids& apic_ids, uint& bit_pos, uint num_values, IdSet& ids) +{ + const uint id_bits = log2(num_values); + if(id_bits == 0) + return; + + const uint mask = bit_mask(id_bits); + + for(size_t i = 0; i < apic_ids.size(); i++) + { + const u8 apic_id = apic_ids[i]; + const u8 field = (apic_id >> bit_pos) & mask; + ids.insert(field); + } + + bit_pos += id_bits; +} + +static int num_packages = -1; // i.e. sockets; > 1 => true SMP system +static int cores_per_package = -1; +static int logical_per_package = -1; // hyperthreading units + +// note: even though APIC IDs are assigned sequentially, we can't make any +// assumptions about the values/ordering because we get them according to +// the CPU affinity mask, which is unknown. + +// HT and dual cores may be disabled, so see how many are actually enabled. +// (examine each APIC ID to determine the kind of CPU) +static void detectActualTopology() +{ + Ids apic_ids; + if(cpu_callByEachCPU(storeApicId, &apic_ids) != INFO::OK) + return; + std::sort(apic_ids.begin(), apic_ids.end()); + debug_assert(std::unique(apic_ids.begin(), apic_ids.end()) == apic_ids.end()); + + const uint max_cores_per_package = coresPerPackage(); + const uint logical_per_core = logicalPerPackage() / max_cores_per_package; + + // extract values from all 3 fields into separate sets + // (we want to know how many logical CPUs per core, cores per package and packages exist) + uint bit_pos = 0; + IdSet logical_ids; + extractFieldsIntoSet(apic_ids, bit_pos, logical_per_core, logical_ids); + IdSet core_ids; + extractFieldsIntoSet(apic_ids, bit_pos, max_cores_per_package, core_ids); + IdSet package_ids; + extractFieldsIntoSet(apic_ids, bit_pos, 0xFF, core_ids); + + num_packages = std::max((int)package_ids.size(), 1); + cores_per_package = std::max((int)core_ids.size(), 1); + logical_per_package = std::max((int)logical_ids.size(), 1) * cores_per_package; +} + + +uint ia32_logicalPerPackage() +{ +#ifndef NDEBUG + debug_assert(logical_per_package != -1); +#endif + return (uint)logical_per_package; +} + +uint ia32_coresPerPackage() +{ +#ifndef NDEBUG + debug_assert(cores_per_package != -1); +#endif + return (uint)cores_per_package; +} + +uint ia32_numPackages() +{ +#ifndef NDEBUG + debug_assert(num_packages != -1); +#endif + return (uint)num_packages; +} + + //----------------------------------------------------------------------------- @@ -676,4 +574,16 @@ LibError ia32_get_call_target(void* ret_addr, void** target) WARN_RETURN(ERR::CPU_UNKNOWN_OPCODE); } -} // extern "C" + +//----------------------------------------------------------------------------- + +void ia32_init() +{ + ia32_asm_cpuid_init(); + + ia32_cap_init(); + + detectVendor(); + + detectActualTopology(); +} diff --git a/source/lib/sysdep/ia32.h b/source/lib/sysdep/ia32/ia32.h similarity index 69% rename from source/lib/sysdep/ia32.h rename to source/lib/sysdep/ia32/ia32.h index 14c06f6524..ad8c48296b 100644 --- a/source/lib/sysdep/ia32.h +++ b/source/lib/sysdep/ia32/ia32.h @@ -27,31 +27,14 @@ #error "including ia32.h without CPU_IA32=1" #endif -// some of these are implemented in asm, so make sure name mangling is -// disabled. -#ifdef __cplusplus -extern "C" { -#endif +#include "ia32_asm.h" +#include "ia32_memcpy.h" // call before any of the following functions extern void ia32_init(); -// -// fast implementations of some sysdep.h functions; see documentation there -// - -extern float ia32_rintf(float f); -extern double ia32_rint(double f); - -extern float ia32_fminf(float f1, float f2); -extern float ia32_fmaxf(float f1, float f2); - -extern i32 ia32_i32_from_float(float f); -extern i32 ia32_i32_from_double(double d); -extern i64 ia32_i64_from_double(double d); - // fpclassify return values #define IA32_FP_NAN 0x0100 #define IA32_FP_NORMAL 0x0400 @@ -59,12 +42,6 @@ extern i64 ia32_i64_from_double(double d); #define IA32_FP_ZERO 0x4000 #define IA32_FP_SUBNORMAL (IA32_FP_NORMAL | IA32_FP_ZERO) -extern uint ia32_fpclassify(double d); -extern uint ia32_fpclassifyf(float f); - -extern void* ia32_memcpy(void* dst, const void* src, size_t nbytes); // asm - - // FPU control word // .. Precision Control: #define IA32_MCW_PC 0x0300 @@ -84,13 +61,15 @@ extern void* ia32_memcpy(void* dst, const void* src, size_t nbytes); // asm #define IA32_EM_UNDERFLOW BIT(4) #define IA32_EM_INEXACT BIT(5) -extern uint ia32_control87(uint new_val, uint mask); // asm -extern u64 ia32_rdtsc_edx_eax(void); +extern void ia32_mfence(); +extern void ia32_serialize(); + + extern u64 ia32_rdtsc_safe(void); #if CONFIG_RETURN64_EDX_EAX -# define ia32_rdtsc ia32_rdtsc_edx_eax +# define ia32_rdtsc ia32_asm_rdtsc_edx_eax #else # define ia32_rdtsc ia32_rdtsc_safe #endif @@ -98,6 +77,16 @@ extern u64 ia32_rdtsc_safe(void); extern void ia32_debug_break(void); +// order in which registers are stored in CPUID regs array +// (do not change! brand string relies on this ordering) +enum IA32Regs +{ + EAX, + EBX, + ECX, + EDX +}; + // CPU capability flags (128 bits) // do not change the order! enum IA32Cap @@ -127,19 +116,6 @@ enum IA32Cap // indicate if the CPU supports the indicated cap. extern bool ia32_cap(IA32Cap cap); - -extern void ia32_get_cpu_info(void); - - -//----------------------------------------------------------------------------- -// internal use only - -// write the current execution state (e.g. all register values) into -// (Win32::CONTEXT*)pcontext (defined as void* to avoid dependency). -extern void ia32_get_current_context(void* pcontext); - -extern void ia32_asm_init(); - // checks if there is an IA-32 CALL instruction right before ret_addr. // returns INFO::OK if so and ERR::FAIL if not. // also attempts to determine the call target. if that is possible @@ -149,23 +125,13 @@ extern void ia32_asm_init(); // this is useful for walking the stack manually. extern LibError ia32_get_call_target(void* ret_addr, void** target); -// order in which registers are stored in regs array -// (do not change! brand string relies on this ordering) -enum IA32Regs -{ - EAX, - EBX, - ECX, - EDX -}; -// try to call the specified CPUID sub-function. returns true on success or -// false on failure (i.e. CPUID or the specific function not supported). -// returns eax, ebx, ecx, edx registers in above order. -extern bool ia32_cpuid(u32 func, u32* regs); +extern const char* ia32_identifierString(); +extern int ia32_isThrottlingPossible(); +extern double ia32_clockFrequency(); -#ifdef __cplusplus -} -#endif +extern uint ia32_logicalPerPackage(); +extern uint ia32_coresPerPackage(); +extern uint ia32_numPackages(); #endif // #ifndef IA32_H diff --git a/source/lib/sysdep/ia32.inc b/source/lib/sysdep/ia32/ia32.inc similarity index 100% rename from source/lib/sysdep/ia32.inc rename to source/lib/sysdep/ia32/ia32.inc diff --git a/source/lib/sysdep/ia32_asm.asm b/source/lib/sysdep/ia32/ia32_asm.asm similarity index 62% rename from source/lib/sysdep/ia32_asm.asm rename to source/lib/sysdep/ia32/ia32_asm.asm index 4ad8cc001b..e741b8e1d1 100644 --- a/source/lib/sysdep/ia32_asm.asm +++ b/source/lib/sysdep/ia32/ia32_asm.asm @@ -19,6 +19,10 @@ %include "ia32.inc" +; note: pure asm functions prevent inlining but also avoid redundant +; store/loads generated by VC inline asm (ugh). + + ;------------------------------------------------------------------------------- ; CPUID support ;------------------------------------------------------------------------------- @@ -34,27 +38,58 @@ max_ext_func dd 0 __SECT__ -; extern "C" bool __cdecl ia32_cpuid(u32 func, u32* regs) -global sym(ia32_cpuid) -sym(ia32_cpuid): +; extern "C" void __cdecl ia32_asm_cpuid_init() +global sym(ia32_asm_cpuid_init) +sym(ia32_asm_cpuid_init): + push ebx + + ; check if CPUID is supported + pushfd + or byte [esp+2], 32 + popfd + pushfd + pop eax + xor edx, edx + shr eax, 22 ; bit 21 toggled? + jnc .no_cpuid + + ; determine max supported CPUID function + xor eax, eax + cpuid + inc eax ; (see max_func decl) + mov [max_func], eax + mov eax, 0x80000000 + cpuid + inc eax ; (see max_func decl) + mov [max_ext_func], eax +.no_cpuid: + + pop ebx + ret + + +; extern "C" bool __cdecl ia32_asm_cpuid(u32 func, u32* regs) +global sym(ia32_asm_cpuid) +sym(ia32_asm_cpuid): push ebx push edi - mov ecx, [esp+8+4+0] ; func + mov edx, [esp+8+4+0] ; func mov edi, [esp+8+4+4] ; -> regs ; compare against max supported func and fail if above + mov ebx, [max_ext_func] xor eax, eax ; return value on failure - test ecx, ecx - mov edx, [max_ext_func] + test edx, edx js .is_ext_func - mov edx, [max_func] + mov ebx, [max_func] .is_ext_func: - cmp ecx, edx + cmp edx, ebx jae .ret ; (see max_func decl) ; issue CPUID and store result registers in array - mov eax, ecx + mov eax, edx + xor ecx, ecx ; CPUID.4 requires ECX = 0..2 cpuid stosd xchg eax, ebx @@ -77,17 +112,12 @@ sym(ia32_cpuid): ; lock-free support routines ;------------------------------------------------------------------------------- -extern sym(cpus) - -; extern "C" void __cdecl atomic_add(intptr_t* location, intptr_t increment); -global sym(atomic_add) -sym(atomic_add): - cmp byte [sym(cpus)], 1 +; extern "C" void __cdecl ia32_asm_atomic_add(intptr_t* location, intptr_t increment); +global sym(ia32_asm_atomic_add) +sym(ia32_asm_atomic_add): mov edx, [esp+4] ; location mov eax, [esp+8] ; increment - je .no_lock db 0xf0 ; LOCK prefix -.no_lock: add [edx], eax ret @@ -102,18 +132,18 @@ db 0xf0 ; LOCK prefix ; bother detecting anything < Pentium, so this'll crash and burn if ; run on 386. we could fall back to simple MOVs there (since 386 CPUs ; aren't MP-capable), but it's not worth the trouble. -; extern "C" __declspec(naked) bool __cdecl CAS_(uintptr_t* location, uintptr_t expected, uintptr_t new_value); -global sym(CAS_) -sym(CAS_): - cmp byte [sym(cpus)], 1 - mov eax, [esp+8] ; expected +; - nor do we bother skipping the LOCK prefix on single-processor systems. +; the branch may be well-predicted, but difference in performance still +; isn't expected to be enough to justify the effort. +; extern "C" ; extern "C" bool __cdecl ia32_asm_CAS(uintptr_t* location, uintptr_t expected, uintptr_t new_value); +global sym(ia32_asm_CAS) +sym(ia32_asm_CAS): mov edx, [esp+4] ; location - cmp edx, 0x10000 ; .. valid pointer? - jb .invalid_location ; no - raise warning + mov eax, [esp+8] ; expected + cmp edx, 0x10000 ; valid location? + jb .invalid_location ; no - raise warning mov ecx, [esp+12] ; new_value - je .no_lock db 0xf0 ; LOCK prefix -.no_lock: cmpxchg [edx], ecx sete al movzx eax, al @@ -131,9 +161,18 @@ db 0xf0 ; LOCK prefix ; FPU ;------------------------------------------------------------------------------- -; extern "C" uint __cdecl ia32_control87(uint new_cw, uint mask); -global sym(ia32_control87) -sym(ia32_control87): +[section .data] + +; to conform with the fallback implementation (a C cast), we need to +; end up with truncate/"chop" rounding. subtracting does the trick, +; assuming RC is the IA-32 default round-to-nearest mode. +round_bias dd 0.4999999 + +__SECT__ + +; extern "C" uint __cdecl ia32_asm_control87(uint new_cw, uint mask); +global sym(ia32_asm_control87) +sym(ia32_asm_control87): push eax fnstcw [esp] pop eax ; old_cw @@ -153,9 +192,9 @@ sym(ia32_control87): ; possible IA-32 FPU control word flags after FXAM: NAN|NORMAL|ZERO FP_CLASSIFY_MASK equ 0x4500 -; extern "C" uint __cdecl ia32_fpclassify(double d); -global sym(ia32_fpclassify) -sym(ia32_fpclassify): +; extern "C" uint __cdecl ia32_asm_fpclassify(double d); +global sym(ia32_asm_fpclassify) +sym(ia32_asm_fpclassify): fld qword [esp+4] fxam fnstsw ax @@ -163,9 +202,9 @@ sym(ia32_fpclassify): and eax, FP_CLASSIFY_MASK ret -; extern "C" uint __cdecl ia32_fpclassifyf(float f); -global sym(ia32_fpclassifyf) -sym(ia32_fpclassifyf): +; extern "C" uint __cdecl ia32_asm_fpclassifyf(float f); +global sym(ia32_asm_fpclassifyf) +sym(ia32_asm_fpclassifyf): fld dword [esp+4] fxam fnstsw ax @@ -174,6 +213,77 @@ sym(ia32_fpclassifyf): ret +; extern "C" float __cdecl ia32_asm_rintf(float) +global sym(ia32_asm_rintf) +sym(ia32_asm_rintf): + fld dword [esp+4] + frndint + ret + +; extern "C" double __cdecl ia32_asm_rint(double) +global sym(ia32_asm_rint) +sym(ia32_asm_rint): + fld qword [esp+4] + frndint + ret + + +; extern "C" float __cdecl ia32_asm_fminf(float, float) +global sym(ia32_asm_fminf) +sym(ia32_asm_fminf): + fld dword [esp+4] + fld dword [esp+8] + fcomi st0, st1 + fcmovnb st0, st1 + fxch + fstp st0 + ret + +; extern "C" float __cdecl ia32_asm_fmaxf(float, float) +global sym(ia32_asm_fmaxf) +sym(ia32_asm_fmaxf): + fld dword [esp+4] + fld dword [esp+8] + fcomi st0, st1 + fcmovb st0, st1 + fxch + fstp st0 + ret + + +; extern "C" i32 __cdecl ia32_asm_i32_from_float(float f) +global sym(ia32_asm_i32_from_float) +sym(ia32_asm_i32_from_float): + push eax + fld dword [esp+8] + fsub dword [round_bias] + fistp dword [esp] + pop eax + ret + +; extern "C" i32 __cdecl ia32_asm_i32_from_double(double d) +global sym(ia32_asm_i32_from_double) +sym(ia32_asm_i32_from_double): + push eax + fld qword [esp+8] + fsub dword [round_bias] + fistp dword [esp] + pop eax + ret + +; extern "C" i64 __cdecl ia32_asm_i64_from_double(double d) +global sym(ia32_asm_i64_from_double) +sym(ia32_asm_i64_from_double): + push edx + push eax + fld qword [esp+12] + fsub dword [round_bias] + fistp qword [esp] + pop eax + pop edx + ret + + ;------------------------------------------------------------------------------- ; misc ;------------------------------------------------------------------------------- @@ -187,9 +297,9 @@ sym(ia32_fpclassifyf): ; xcode complains about CPUID clobbering ebx, so we use external asm ; where possible (IA-32 CPUs). ; -; extern "C" u64 ia32_rdtsc_edx_eax() -global sym(ia32_rdtsc_edx_eax) -sym(ia32_rdtsc_edx_eax): +; extern "C" u64 ia32_asm_rdtsc_edx_eax() +global sym(ia32_asm_rdtsc_edx_eax) +sym(ia32_asm_rdtsc_edx_eax): push ebx cpuid pop ebx @@ -199,12 +309,12 @@ sym(ia32_rdtsc_edx_eax): ; write the current execution state (e.g. all register values) into ; (Win32::CONTEXT*)pcontext (defined as void* to avoid dependency). -; optimized for size; this must be straight asm because __declspec(naked) +; optimized for size; this must be straight asm because ; extern "C" ; is compiler-specific and compiler-generated prolog code inserted before ; inline asm trashes EBP and ESP (unacceptable). -; extern "C" void ia32_get_current_context(void* pcontext) -global sym(ia32_get_current_context) -sym(ia32_get_current_context): +; extern "C" void ia32_asm_get_current_context(void* pcontext) +global sym(ia32_asm_get_current_context) +sym(ia32_asm_get_current_context): pushad pushfd mov edi, [esp+4+32+4] ; pcontext @@ -269,37 +379,3 @@ rep stosd popad ret - - -;------------------------------------------------------------------------------- -; init -;------------------------------------------------------------------------------- - -; extern "C" bool __cdecl ia32_asm_init() -global sym(ia32_asm_init) -sym(ia32_asm_init): - push ebx - - ; check if CPUID is supported - pushfd - or byte [esp+2], 32 - popfd - pushfd - pop eax - xor edx, edx - shr eax, 22 ; bit 21 toggled? - jnc .no_cpuid - - ; determine max supported CPUID function - xor eax, eax - cpuid - inc eax ; (see max_func decl) - mov [max_func], eax - mov eax, 0x80000000 - cpuid - inc eax ; (see max_func decl) - mov [max_ext_func], eax -.no_cpuid: - - pop ebx - ret diff --git a/source/lib/sysdep/ia32/ia32_asm.h b/source/lib/sysdep/ia32/ia32_asm.h new file mode 100644 index 0000000000..ca395071f7 --- /dev/null +++ b/source/lib/sysdep/ia32/ia32_asm.h @@ -0,0 +1,44 @@ +#ifndef INCLUDED_IA32_ASM +#define INCLUDED_IA32_ASM + +#ifdef __cplusplus +extern "C" { +#endif + +extern void ia32_asm_cpuid_init(); + +// try to call the specified CPUID sub-function. returns true on success or +// false on failure (i.e. CPUID or the specific function not supported). +// returns eax, ebx, ecx, edx registers in above order. +extern bool ia32_asm_cpuid(u32 func, u32* regs); + +extern void ia32_asm_atomic_add(intptr_t* location, intptr_t increment); + + +extern bool ia32_asm_CAS(uintptr_t* location, uintptr_t expected, uintptr_t new_value); + +extern uint ia32_asm_control87(uint new_val, uint mask); + +extern uint ia32_asm_fpclassify(double d); +extern uint ia32_asm_fpclassifyf(float f); + +extern float ia32_asm_rintf(float); +extern double ia32_asm_rint(double); +extern float ia32_asm_fminf(float, float); +extern float ia32_asm_fmaxf(float, float); + +extern i32 ia32_asm_i32_from_float(float f); +extern i32 ia32_asm_i32_from_double(double d); +extern i64 ia32_asm_i64_from_double(double d); + +extern u64 ia32_asm_rdtsc_edx_eax(void); + +// write the current execution state (e.g. all register values) into +// (Win32::CONTEXT*)pcontext (defined as void* to avoid dependency). +extern void ia32_asm_get_current_context(void* pcontext); + +#ifdef __cplusplus +} +#endif + +#endif // #ifndef INCLUDED_IA32_ASM diff --git a/source/lib/sysdep/ia32_memcpy.asm b/source/lib/sysdep/ia32/ia32_memcpy.asm similarity index 100% rename from source/lib/sysdep/ia32_memcpy.asm rename to source/lib/sysdep/ia32/ia32_memcpy.asm diff --git a/source/lib/sysdep/ia32/ia32_memcpy.h b/source/lib/sysdep/ia32/ia32_memcpy.h new file mode 100644 index 0000000000..cfa79329af --- /dev/null +++ b/source/lib/sysdep/ia32/ia32_memcpy.h @@ -0,0 +1,15 @@ +#ifndef INCLUDED_IA32_MEMCPY +#define INCLUDED_IA32_MEMCPY + +#include "lib/sysdep/sysdep.h" + +extern "C" { + +extern void ia32_memcpy_init(); + +#define RESTRICT __restrict +extern void* ia32_memcpy(void* RESTRICT dst, const void* RESTRICT src, size_t nbytes); + +} + +#endif // #ifndef INCLUDED_IA32_MEMCPY diff --git a/source/lib/sysdep/ia32/ia32_memcpy_init.cpp b/source/lib/sysdep/ia32/ia32_memcpy_init.cpp new file mode 100644 index 0000000000..c73c64393a --- /dev/null +++ b/source/lib/sysdep/ia32/ia32_memcpy_init.cpp @@ -0,0 +1,22 @@ +#include "precompiled.h" + +#include "ia32.h" +#include "ia32_memcpy.h" + +// set by ia32_memcpy_init, referenced by ia32_memcpy (asm) +// default to "all codepaths supported" +extern "C" u32 ia32_memcpy_size_mask = ~0u; + +void ia32_memcpy_init() +{ + // set the mask that is applied to transfer size before + // choosing copy technique. this is the mechanism for disabling + // codepaths that aren't supported on all CPUs; see article for details. + // .. check for PREFETCHNTA and MOVNTQ support. these are part of the SSE + // instruction set, but also supported on older Athlons as part of + // the extended AMD MMX set. + if(!ia32_cap(IA32_CAP_SSE) && !ia32_cap(IA32_CAP_AMD_MMX_EXT)) + ia32_memcpy_size_mask = 0u; +} + +// ia32_memcpy() is defined in ia32_memcpy_asm.asm diff --git a/source/lib/sysdep/sysdep.cpp b/source/lib/sysdep/sysdep.cpp index 003cc704c2..3c653578e9 100644 --- a/source/lib/sysdep/sysdep.cpp +++ b/source/lib/sysdep/sysdep.cpp @@ -68,45 +68,3 @@ uint fpclassifyf(float f) #endif #endif // #if !HAVE_C99_MATH - - -// float->int conversion: not using the ia32 version; just implement as a -// cast. (see USE_IA32_FLOAT_TO_INT definition for details) -#if !USE_IA32_FLOAT_TO_INT - -i32 i32_from_float(float f) -{ - return (i32)f; -} - -i32 i32_from_double(double d) -{ - return (i32)d; -} - -i64 i64_from_double(double d) -{ - return (i64)d; -} - -#endif - -#if !HAVE_STRDUP - -char* strdup(const char* str) -{ - const size_t num_chars = strlen(str); - char* new_str = malloc(num_chars*sizeof(char)+1); - SAFE_STRCPY(new_str, str); - return new_str; -} - -wchar_t* wcsdup(const wchar_t* str) -{ - const size_t num_chars = wcslen(str); - wchar_t* new_str = malloc(num_chars*sizeof(wchar_t)+1); - SAFE_WCSCPY(new_str, str); - return new_str; -} - -#endif // #if !HAVE_STRDUP diff --git a/source/lib/sysdep/sysdep.h b/source/lib/sysdep/sysdep.h index a9d43f86b4..cd493a5b78 100644 --- a/source/lib/sysdep/sysdep.h +++ b/source/lib/sysdep/sysdep.h @@ -40,7 +40,7 @@ # include "lib/sysdep/unix/unix.h" #endif #if CPU_IA32 -#include "ia32.h" +#include "ia32/ia32.h" #endif @@ -109,17 +109,17 @@ extern void* alloca(size_t size); #if !HAVE_C99_MATH // .. fast IA-32 versions # if CPU_IA32 -# define rintf ia32_rintf -# define rint ia32_rint -# define fminf ia32_fminf -# define fmaxf ia32_fmaxf +# define rintf ia32_asm_rintf +# define rint ia32_asm_rint +# define fminf ia32_asm_fminf +# define fmaxf ia32_asm_fmaxf # define FP_NAN IA32_FP_NAN # define FP_NORMAL IA32_FP_NORMAL # define FP_INFINITE (FP_NAN | FP_NORMAL) # define FP_ZERO IA32_FP_ZERO # define FP_SUBNORMAL (FP_NORMAL | FP_ZERO) -# define fpclassify(x) ( (sizeof(x) == sizeof(float))? ia32_fpclassifyf(x) : ia32_fpclassify(x) ) +# define fpclassify(x) ( (sizeof(x) == sizeof(float))? ia32_asm_fpclassifyf(x) : ia32_asm_fpclassify(x) ) // .. portable C emulation # else extern float rintf(float f); @@ -153,6 +153,7 @@ extern void* alloca(size_t size); # endif #endif // HAVE_C99_MATH + // C99-like restrict (non-standard in C++, but widely supported in various forms). // // May be used on pointers. May also be used on member functions to indicate @@ -195,6 +196,7 @@ extern void* alloca(size_t size); # define RESTRICT #endif + // C99 __func__ // .. already available; need do nothing #if HAVE_C99 @@ -206,11 +208,6 @@ extern void* alloca(size_t size); # define __func__ "(unknown)" #endif -#if !HAVE_STRDUP -extern char* strdup(const char* str); -extern wchar_t* wcsdup(const wchar_t* str); -#endif // #if !HAVE_STRDUP - //----------------------------------------------------------------------------- // sysdep API @@ -309,42 +306,8 @@ extern LibError sys_get_executable_name(char* n_path, size_t buf_size); // PATH_MAX chars. extern LibError sys_pick_directory(char* n_path, size_t buf_size); -// execute the specified function once on each CPU. -// this includes logical HT units and proceeds serially (function -// is never re-entered) in order of increasing OS CPU ID. -// note: implemented by switching thread affinity masks and forcing -// a reschedule, which is apparently not possible with POSIX. -// -// may fail if e.g. OS is preventing us from running on some CPUs. -// called from ia32.cpp get_cpu_count. -extern LibError sys_on_each_cpu(void (*cb)()); -// drop-in replacement for libc memcpy(). only requires CPU support for -// MMX (by now universal). highly optimized for Athlon and Pentium III -// microarchitectures; significantly outperforms VC7.1 memcpy and memcpy_amd. -// for details, see accompanying article. -#ifdef CPU_IA32 -# define memcpy2 ia32_memcpy -extern void* ia32_memcpy(void* dst, const void* src, size_t nbytes); -#else -# define memcpy2 memcpy -#endif - -// i32_from_float et al: convert float to int. much faster than _ftol2, -// which would normally be used by (int) casts. -// .. fast IA-32 version: only used in some cases; see macro definition. -#if USE_IA32_FLOAT_TO_INT -# define i32_from_float ia32_i32_from_float -# define i32_from_double ia32_i32_from_double -# define i64_from_double ia32_i64_from_double -// .. portable C emulation -#else -extern i32 i32_from_float(float); -extern i32 i32_from_double(double); -extern i64 i64_from_double(double); -#endif - // return the largest sector size [bytes] of any storage medium // (HD, optical, etc.) in the system. diff --git a/source/lib/sysdep/tests/test_sysdep.h b/source/lib/sysdep/tests/test_sysdep.h index 24128c08a0..036c319cfc 100644 --- a/source/lib/sysdep/tests/test_sysdep.h +++ b/source/lib/sysdep/tests/test_sysdep.h @@ -8,20 +8,20 @@ class TestSysdep : public CxxTest::TestSuite public: void test_float_int() { - TS_ASSERT_EQUALS(i32_from_float(0.99999f), 0); - TS_ASSERT_EQUALS(i32_from_float(1.0f), 1); - TS_ASSERT_EQUALS(i32_from_float(1.01f), 1); - TS_ASSERT_EQUALS(i32_from_float(5.6f), 5); + TS_ASSERT_EQUALS(cpu_i32_from_float(0.99999f), 0); + TS_ASSERT_EQUALS(cpu_i32_from_float(1.0f), 1); + TS_ASSERT_EQUALS(cpu_i32_from_float(1.01f), 1); + TS_ASSERT_EQUALS(cpu_i32_from_float(5.6f), 5); - TS_ASSERT_EQUALS(i32_from_double(0.99999), 0); - TS_ASSERT_EQUALS(i32_from_double(1.0), 1); - TS_ASSERT_EQUALS(i32_from_double(1.01), 1); - TS_ASSERT_EQUALS(i32_from_double(5.6), 5); + TS_ASSERT_EQUALS(cpu_i32_from_double(0.99999), 0); + TS_ASSERT_EQUALS(cpu_i32_from_double(1.0), 1); + TS_ASSERT_EQUALS(cpu_i32_from_double(1.01), 1); + TS_ASSERT_EQUALS(cpu_i32_from_double(5.6), 5); - TS_ASSERT_EQUALS(i64_from_double(0.99999), 0LL); - TS_ASSERT_EQUALS(i64_from_double(1.0), 1LL); - TS_ASSERT_EQUALS(i64_from_double(1.01), 1LL); - TS_ASSERT_EQUALS(i64_from_double(5.6), 5LL); + TS_ASSERT_EQUALS(cpu_i64_from_double(0.99999), 0LL); + TS_ASSERT_EQUALS(cpu_i64_from_double(1.0), 1LL); + TS_ASSERT_EQUALS(cpu_i64_from_double(1.01), 1LL); + TS_ASSERT_EQUALS(cpu_i64_from_double(5.6), 5LL); } void test_round() diff --git a/source/lib/sysdep/unix/bsd.cpp b/source/lib/sysdep/unix/bsd.cpp new file mode 100644 index 0000000000..65e77f9414 --- /dev/null +++ b/source/lib/sysdep/unix/bsd.cpp @@ -0,0 +1,24 @@ +#include "precompiled.h" +#include "bsd.h" + + +static int sysctlFromMemType(CpuMemoryIndicators mem_type) +{ + switch(mem_type) + { + case MEM_TOTAL: + return HW_PHYSMEM; + case MEM_AVAILABLE: + return HW_USERMEM; + } + UNREACHABLE; +} + +size_t bsd_memorySize(CpuMemoryIndicators mem_type) +{ + size_t memory_size = 0; + size_t len = sizeof(memory_size); + const int mib[2] = { CTL_HW, sysctlFromMemType(mem_type) }; + sysctl(mib, 2, &memory_size, &len, 0, 0); + return memory_size; +} diff --git a/source/lib/sysdep/unix/bsd.h b/source/lib/sysdep/unix/bsd.h new file mode 100644 index 0000000000..21d19c7f8a --- /dev/null +++ b/source/lib/sysdep/unix/bsd.h @@ -0,0 +1,8 @@ +#ifndef INCLUDED_BSD +#define INCLUDED_BSD + +#include "../cpu.h" + +extern size_t bsd_memorySize(CpuMemoryIndicators mem_type); + +#endif // #ifndef INCLUDED_BSD diff --git a/source/lib/sysdep/unix/ucpu.cpp b/source/lib/sysdep/unix/ucpu.cpp new file mode 100644 index 0000000000..523180fd5f --- /dev/null +++ b/source/lib/sysdep/unix/ucpu.cpp @@ -0,0 +1,52 @@ +#include "precompiled.h" + +#include "ucpu.h" + +int ucpu_isThrottlingPossible() +{ + return -1; // don't know +} + +int ucpu_numPackages() +{ + long res = sysconf(_SC_NPROCESSORS_CONF); + if (res == -1) + return 1; + else + return (int)res; +} + +double ucpu_clockFrequency() +{ + return -1; // don't know +} + + +// apparently not possible on non-Windows OSes because they seem to lack +// a CPU affinity API. see sysdep.h comment. +LibError ucpu_callByEachCPU(CpuCallback cb, void* param) +{ + UNUSED2(cb); + +/* +cpu_set_t currentCPU; + +while ( j < sysNumProcs ) + +{ + +CPU_ZERO(¤tCPU); + +CPU_SET(j, ¤tCPU); + +if ( sched_setaffinity (0, sizeof (currentCPU), ¤tCPU) + +== 0 ) + +{ + +sleep(0); // Ensure system to switch to the right +*/ + + return ERR::NO_SYS; +} diff --git a/source/lib/sysdep/unix/ucpu.h b/source/lib/sysdep/unix/ucpu.h new file mode 100644 index 0000000000..138f465786 --- /dev/null +++ b/source/lib/sysdep/unix/ucpu.h @@ -0,0 +1,11 @@ +#ifndef INCLUDED_UCPU +#define INCLUDED_UCPU + +#include "lib/sysdep/cpu.h" + +extern int ucpu_isThrottlingPossible(); +extern int ucpu_numPackages(); +extern double ucpu_clockFrequency(); +extern LibError ucpu_callByEachCPU(CpuCallback cb, void* param); + +#endif // #ifndef INCLUDED_UCPU diff --git a/source/lib/sysdep/unix/unix.cpp b/source/lib/sysdep/unix/unix.cpp index ae4e76e837..f1ff74553f 100644 --- a/source/lib/sysdep/unix/unix.cpp +++ b/source/lib/sysdep/unix/unix.cpp @@ -41,25 +41,6 @@ LibError sys_get_executable_name(char* n_path, size_t buf_size) return INFO::OK; } -extern int cpus; -int unix_get_cpu_info() -{ - long res = sysconf(_SC_NPROCESSORS_CONF); - if (res == -1) - cpus = 1; - else - cpus = (int)res; - return 0; -} - -// apparently not possible on non-Windows OSes because they seem to lack -// a CPU affinity API. see sysdep.h comment. -LibError sys_on_each_cpu(void(*cb)()) -{ - UNUSED2(cb); - - return ERR::NO_SYS; -} ErrorReaction sys_display_error(const wchar_t* text, uint flags) { diff --git a/source/lib/sysdep/win/delay_load.cpp b/source/lib/sysdep/win/delay_load.cpp index fe14f7c003..8c1e7b0667 100644 --- a/source/lib/sysdep/win/delay_load.cpp +++ b/source/lib/sysdep/win/delay_load.cpp @@ -26,6 +26,7 @@ #include "delay_load.h" #include "win_internal.h" +#include "lib/sysdep/cpu.h" #define _DELAY_IMP_VER 2 @@ -261,7 +262,7 @@ PinhFromImageBase(HMODULE hmod) { static inline void WINAPI OverlayIAT(PImgThunkData pitdDst, PCImgThunkData pitdSrc) { - memcpy2(pitdDst, pitdSrc, CountOfImports(pitdDst) * sizeof IMAGE_THUNK_DATA); + cpu_memcpy(pitdDst, pitdSrc, CountOfImports(pitdDst) * sizeof IMAGE_THUNK_DATA); } static inline DWORD WINAPI diff --git a/source/lib/sysdep/win/tests/test_ia32.h b/source/lib/sysdep/win/tests/test_ia32.h index 656ba32265..c5c26be4c9 100644 --- a/source/lib/sysdep/win/tests/test_ia32.h +++ b/source/lib/sysdep/win/tests/test_ia32.h @@ -1,7 +1,7 @@ #include "lib/self_test.h" #include "lib/lib.h" -#include "lib/sysdep/ia32.h" +#include "lib/sysdep/ia32/ia32.h" // note: ia32_i??_from_*, ia32_rint*, ia32_fm??f are all tested within // sysdep to avoid test duplication (both the ia32 versions and diff --git a/source/lib/sysdep/win/wcpu.cpp b/source/lib/sysdep/win/wcpu.cpp index d7596a6523..7922312c88 100644 --- a/source/lib/sysdep/win/wcpu.cpp +++ b/source/lib/sysdep/win/wcpu.cpp @@ -21,7 +21,7 @@ */ #include "precompiled.h" -#include "lib/sysdep/cpu.h" +#include "wcpu.h" #include "lib/lib.h" #include "lib/posix/posix_pthread.h" @@ -29,17 +29,31 @@ #include "win_internal.h" // limit allows statically allocated per-CPU structures (for simplicity). -// we're Windows-specific anyway; such systems won't foreseeably have more. -// note: int instead of unsigned because is also signed (tri-state). +// WinAPI only supports max. 32 CPUs anyway (due to DWORD bitfields). +// signed int because num_cpus is tri-state. static const int MAX_CPUS = 32; -static void check_speedstep() + +int wcpu_numProcessors() +{ + // get number of CPUs (can't fail) + SYSTEM_INFO si; + GetSystemInfo(&si); + const int num_processors = (int)si.dwNumberOfProcessors; + return num_processors; +} + + +int wcpu_isThrottlingPossible() { WIN_SAVE_LAST_ERROR; + int is_throttling_possible = -1; // CallNtPowerInformation // (manual import because it's not supported on Win95) NTSTATUS (WINAPI *pCNPI)(POWER_INFORMATION_LEVEL, PVOID, ULONG, PVOID, ULONG) = 0; + // this is most likely the only reference, so don't free it + // (=> unload) until done with the DLL. HMODULE hPowrprofDll = LoadLibrary("powrprof.dll"); *(void**)&pCNPI = GetProcAddress(hPowrprofDll, "CallNtPowerInformation"); if(pCNPI) @@ -47,77 +61,58 @@ static void check_speedstep() // most likely not speedstep-capable if these aren't supported SYSTEM_POWER_CAPABILITIES spc; if(pCNPI(SystemPowerCapabilities, 0,0, &spc,sizeof(spc)) == STATUS_SUCCESS) + { if(!spc.ProcessorThrottle || !spc.ThermalControl) - cpu_speedstep = 0; + is_throttling_possible = 0; + } // probably speedstep if cooling mode active. // the documentation of PO_TZ_* is unclear, so we can't be sure. SYSTEM_POWER_INFORMATION spi; if(pCNPI(SystemPowerInformation, 0,0, &spi,sizeof(spi)) == STATUS_SUCCESS) + { if(spi.CoolingMode != PO_TZ_INVALID_MODE) - cpu_speedstep = 1; + is_throttling_possible = 1; + } // definitely speedstep if any throttle is less than 100%. PROCESSOR_POWER_INFORMATION ppi[MAX_CPUS]; if(pCNPI(ProcessorInformation, 0,0, ppi,sizeof(ppi)) == STATUS_SUCCESS) { const PROCESSOR_POWER_INFORMATION* p = ppi; - for(int i = 0; i < MIN(cpus, MAX_CPUS); i++, p++) + for(int i = 0; i < MIN(wcpu_numProcessors(), MAX_CPUS); i++, p++) + { if(p->MhzLimit != p->MaxMhz || p->CurrentMhz != p->MaxMhz) { - cpu_speedstep = 1; + is_throttling_possible = 1; break; } + } } } FreeLibrary(hPowrprofDll); - // this is most likely the only reference, - // so don't free it (=> unload) until done with the DLL. // CallNtPowerInformation not available, or none of the above apply: // don't know yet (for certain, at least). - if(cpu_speedstep == -1) + if(is_throttling_possible == -1) { // check if running on a laptop HW_PROFILE_INFO hi; GetCurrentHwProfile(&hi); - bool is_laptop = !(hi.dwDockInfo & DOCKINFO_DOCKED) ^ !(hi.dwDockInfo & DOCKINFO_UNDOCKED); + const bool is_laptop = !(hi.dwDockInfo & DOCKINFO_DOCKED) ^ !(hi.dwDockInfo & DOCKINFO_UNDOCKED); // both flags set <==> this is a desktop machine. // both clear is unspecified; we assume it's not a laptop. // NOTE: ! is necessary (converts expression to bool) // we'll guess SpeedStep is active if on a laptop. // ia32 code will get a second crack at it. - cpu_speedstep = (is_laptop)? 1 : 0; + is_throttling_possible = (is_laptop)? 1 : 0; } - + WIN_RESTORE_LAST_ERROR; -} - -LibError win_get_cpu_info() -{ - // get number of CPUs (can't fail) - SYSTEM_INFO si; - GetSystemInfo(&si); - cpus = (int)si.dwNumberOfProcessors; - - // read CPU frequency from registry - HKEY hKey; - const char* key = "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"; - if(RegOpenKeyEx(HKEY_LOCAL_MACHINE, key, 0, KEY_QUERY_VALUE, &hKey) == 0) - { - DWORD freq_mhz; - DWORD size = sizeof(freq_mhz); - if(RegQueryValueEx(hKey, "~MHz", 0, 0, (LPBYTE)&freq_mhz, &size) == 0) - cpu_freq = freq_mhz * 1e6; - - RegCloseKey(hKey); - } - - check_speedstep(); - - return INFO::OK; + debug_assert(is_throttling_possible == 0 || is_throttling_possible == 1); + return is_throttling_possible; } @@ -131,7 +126,7 @@ LibError win_get_cpu_info() // // may fail if e.g. OS is preventing us from running on some CPUs. // called from ia32.cpp get_cpu_count. -LibError sys_on_each_cpu(void (*cb)()) +LibError wcpu_callByEachCPU(CpuCallback cb, void* param) { const HANDLE hProcess = GetCurrentProcess(); DWORD process_affinity, system_affinity; @@ -142,26 +137,26 @@ LibError sys_on_each_cpu(void (*cb)()) if(process_affinity != system_affinity) WARN_RETURN(ERR::CPU_RESTRICTED_AFFINITY); - for(DWORD cpu_bit = 1; cpu_bit != 0 && cpu_bit <= process_affinity; cpu_bit *= 2) + for(DWORD_PTR cpu_bit = 1; cpu_bit != 0 && cpu_bit <= process_affinity; cpu_bit *= 2) { // check if we can switch to target CPU if(!(process_affinity & cpu_bit)) continue; // .. and do so. - if(!SetProcessAffinityMask(hProcess, process_affinity)) + if(!SetThreadAffinityMask(GetCurrentThread(), cpu_bit)) { WARN_ERR(ERR::CPU_RESTRICTED_AFFINITY); continue; } - // reschedule, to make sure we switch CPUs + // reschedule to make sure we switch CPUs. Sleep(1); - cb(); + cb(param); } // restore to original value - SetProcessAffinityMask(hProcess, process_affinity); + SetThreadAffinityMask(hProcess, process_affinity); return INFO::OK; } @@ -388,4 +383,4 @@ probably hard; a start would be just the function in which the address is, then ========================================== -*/ \ No newline at end of file +*/ diff --git a/source/lib/sysdep/win/wcpu.h b/source/lib/sysdep/win/wcpu.h new file mode 100644 index 0000000000..6f77299d38 --- /dev/null +++ b/source/lib/sysdep/win/wcpu.h @@ -0,0 +1,11 @@ +#ifndef INCLUDED_WCPU +#define INCLUDED_WCPU + +#include "lib/sysdep/cpu.h" + +extern int wcpu_numProcessors(); +extern int wcpu_isThrottlingPossible(); +extern double wcpu_clockFrequency(); +extern LibError wcpu_callByEachCPU(CpuCallback cb, void* param); + +#endif // #ifndef INCLUDED_WCPU diff --git a/source/lib/sysdep/win/wdbg_sym.cpp b/source/lib/sysdep/win/wdbg_sym.cpp index fd5f31bcfe..b95e4c3af1 100644 --- a/source/lib/sysdep/win/wdbg_sym.cpp +++ b/source/lib/sysdep/win/wdbg_sym.cpp @@ -34,7 +34,7 @@ #include "lib/app_hooks.h" #include "lib/path_util.h" #if CPU_IA32 -# include "lib/sysdep/ia32.h" +# include "lib/sysdep/ia32/ia32.h" #endif #include "win_internal.h" @@ -369,7 +369,7 @@ static LibError walk_stack(StackFrameCallback cb, void* user_arg = 0, uint skip // compiler-generated prolog code trashes some registers. #if CPU_IA32 - ia32_get_current_context(&context); + ia32_asm_get_current_context(&context); #else // preferred implementation (was imported during module init) if(pRtlCaptureContext) diff --git a/source/lib/sysdep/win/win.cpp b/source/lib/sysdep/win/win.cpp index 73d97af82d..8bd4edef6c 100644 --- a/source/lib/sysdep/win/win.cpp +++ b/source/lib/sysdep/win/win.cpp @@ -27,6 +27,7 @@ #include "win_internal.h" #include "lib/path_util.h" +#include "lib/posix/posix.h" #if MSC_VERSION >= 1400 #include // __security_init_cookie @@ -37,7 +38,6 @@ char win_sys_dir[MAX_PATH+1]; char win_exe_dir[MAX_PATH+1]; - // only call after a Win32 function indicates failure. static LibError LibError_from_GLE(bool warn_if_failed = true) { diff --git a/source/lib/sysdep/win/win_internal.h b/source/lib/sysdep/win/win_internal.h index f934e10d9f..370a26aec8 100644 --- a/source/lib/sysdep/win/win_internal.h +++ b/source/lib/sysdep/win/win_internal.h @@ -430,6 +430,10 @@ extern int win_is_locked(uint idx); // cope with split string literals (e.g. "ab" "c"). that disqualifies // it, since we want to hide the section name behind a macro, which // would require the abovementioned merging. + +// note: the purpose of pre-libc init (with the resulting requirement that +// no CRT functions be used during init!) is to allow the use of the +// initialized module in static ctors. #define SECTION_PRE_LIBC(group) data_seg(".LIB$C" #group) #define SECTION_PRE_MAIN(group) data_seg(".LIB$I" #group) #define SECTION_POST_ATEXIT(group) data_seg(".LIB$T" #group) diff --git a/source/lib/sysdep/win/wposix/waio.cpp b/source/lib/sysdep/win/wposix/waio.cpp index 9ba5a9bfc6..6252b5b672 100644 --- a/source/lib/sysdep/win/wposix/waio.cpp +++ b/source/lib/sysdep/win/wposix/waio.cpp @@ -30,7 +30,7 @@ #include "wfilesystem.h" // mode_t #include "wtime.h" // timespec #include "waio_internal.h" - +#include "lib/sysdep/cpu.h" #pragma SECTION_PRE_LIBC(J) WIN_REGISTER_FUNC(waio_init); @@ -657,7 +657,7 @@ static int aio_rw(struct aiocb* cb) // unaligned buffer: copy to align buffer and write from there. if(buf_misaligned) { - memcpy2(r->buf, buf, size); + cpu_memcpy(r->buf, buf, size); actual_buf = r->buf; // clear previous contents at end of align buf memset((char*)r->buf + size, 0, actual_size - size); @@ -761,7 +761,7 @@ ssize_t aio_return(struct aiocb* cb) // we read into align buffer - copy to user's buffer if(r->read_into_align_buffer) - memcpy2((void*)cb->aio_buf, (u8*)r->buf + r->pad, cb->aio_nbytes); + cpu_memcpy((void*)cb->aio_buf, (u8*)r->buf + r->pad, cb->aio_nbytes); // TODO: this copies data back into original buffer from align buffer // when writing from unaligned buffer. unnecessarily slow. diff --git a/source/lib/sysdep/win/wposix/wfilesystem.cpp b/source/lib/sysdep/win/wposix/wfilesystem.cpp index bea815c18b..749cb1420c 100644 --- a/source/lib/sysdep/win/wposix/wfilesystem.cpp +++ b/source/lib/sysdep/win/wposix/wfilesystem.cpp @@ -140,11 +140,20 @@ int stat(const char* fn, struct stat* s) */ -int mkdir(const char* path, mode_t) +#if !HAVE_MKDIR + +int mkdir(const char* path, mode_t UNUSED(mode)) { - return CreateDirectory(path, 0)? 0 : -1; + if(!CreateDirectory(path, (LPSECURITY_ATTRIBUTES)NULL)) + { + return -1; + } + + return 0; } +#endif // #if !HAVE_MKDIR + //----------------------------------------------------------------------------- // readdir diff --git a/source/lib/sysdep/win/wposix/wtime.cpp b/source/lib/sysdep/win/wposix/wtime.cpp index c63714ef2e..d3b04065fa 100644 --- a/source/lib/sysdep/win/wposix/wtime.cpp +++ b/source/lib/sysdep/win/wposix/wtime.cpp @@ -29,7 +29,7 @@ #include #include "lib/adts.h" -#include "lib/sysdep/ia32.h" +#include "lib/sysdep/ia32/ia32.h" #include "lib/sysdep/cpu.h" #include "wposix_internal.h" #include "wpthread.h" @@ -77,7 +77,7 @@ AT_STARTUP(\ // (default values for HRT_NONE impl) // initial measurement of the time source's tick rate. not necessarily -// correct (e.g. when using TSC; cpu_freq isn't exact). +// correct (e.g. when using TSC; cpu_clockFrequency isn't exact). static double hrt_nominal_freq = -1.0; // actual resolution of the time source (may differ from hrt_nominal_freq @@ -199,14 +199,14 @@ static LibError choose_impl() // will do this as well (if not to save power, for heat reasons). // frequency changes are too often and drastic to correct, // and we don't want to mess with the system power settings => unsafe. - if(cpu_freq > 0.0 && ia32_cap(IA32_CAP_TSC)) + if(cpu_isModuleInitialized() && cpu_clockFrequency() > 0.0 && ia32_cap(IA32_CAP_TSC)) { - safe = (cpus == 1 && cpu_speedstep == 0); + safe = (cpu_coresPerPackage() == 1 && cpu_numPackages() == 1 && cpu_isThrottlingPossible() == 0); SAFETY_OVERRIDE(HRT_TSC); if(safe) { hrt_impl = HRT_TSC; - hrt_nominal_freq = cpu_freq; + hrt_nominal_freq = cpu_clockFrequency(); hrt_res = (1.0 / hrt_nominal_freq); return INFO::OK; } @@ -249,13 +249,13 @@ static LibError choose_impl() else { // can't decide yet - assume unsafe - if(cpu_freq == 0.0) + if(!cpu_isModuleInitialized()) safe = false; else { // compare QPC freq to CPU clock freq - can't rule out HPET, // because its frequency isn't known (it's at least 10 MHz). - double freq_dist = fabs(cpu_freq/qpc_freq - 1.0); + double freq_dist = fabs(cpu_clockFrequency()/qpc_freq - 1.0); safe = freq_dist > 0.05; // safe if freqs not within 5% (i.e. it doesn't use TSC) } @@ -711,7 +711,7 @@ static i64 time_ns() } const double dt = hrt_time() - hrt_start_time; - const i64 ns = st_start + i64_from_double(dt * _1e9); + const i64 ns = st_start + cpu_i64_from_double(dt * _1e9); return ns; } diff --git a/source/lib/tests/test_lockfree.h b/source/lib/tests/test_lockfree.h index 6db9fcaee6..5545efd184 100644 --- a/source/lib/tests/test_lockfree.h +++ b/source/lib/tests/test_lockfree.h @@ -88,7 +88,7 @@ class TestMultithread : public CxxTest::TestSuite TestMultithread* this_ = param->this_; const uintptr_t thread_number = param->thread_number; - atomic_add(&this_->num_active_threads, 1); + cpu_atomic_add(&this_->num_active_threads, 1); // chosen randomly every iteration (int_value % 4) enum TestAction @@ -176,7 +176,7 @@ class TestMultithread : public CxxTest::TestSuite } // switch } // while !is_complete - atomic_add(&this_->num_active_threads, -1); + cpu_atomic_add(&this_->num_active_threads, -1); TS_ASSERT(this_->num_active_threads >= 0); delete param; diff --git a/source/lib/timer.cpp b/source/lib/timer.cpp index ef4c0d3c83..9ac37784ae 100644 --- a/source/lib/timer.cpp +++ b/source/lib/timer.cpp @@ -307,7 +307,7 @@ void timer_display_client_totals() double sum; #if TIMER_USE_RAW_TICKS # if CPU_IA32 - sum = tc->sum / cpu_freq; + sum = tc->sum / cpu_clockFrequency(); # else # error "port" # endif diff --git a/source/lib/timer.h b/source/lib/timer.h index 469a628f3f..3ff480df09 100644 --- a/source/lib/timer.h +++ b/source/lib/timer.h @@ -70,12 +70,13 @@ extern void calc_fps(void); // // note that overflow isn't an issue either way (63 bit cycle counts // at 10 GHz cover intervals of 29 years). -#define TIMER_USE_RAW_TICKS 1 -#if TIMER_USE_RAW_TICKS +#if CPU_IA32 +# define TIMER_USE_RDTSC 1 typedef i64 TimerUnit; #else +# define TIMER_USE_RDTSC 0 typedef double TimerUnit; -#endif +#endif // #if CPU_IA32 // opaque - do not access its fields! // note: must be defined here because clients instantiate them; @@ -192,9 +193,8 @@ Example usage: #define TIMER_END(description) } -#if TIMER_USE_RAW_TICKS +#if TIMER_USE_RDTSC -#if CPU_IA32 // fast, not usable as wall-clock (http://www.gamedev.net/reference/programming/features/timing) class TimerRdtsc { @@ -205,9 +205,6 @@ public: return ia32_rdtsc(); } }; -#else -# error "port" -#endif // CPU_IA32 #else @@ -221,7 +218,7 @@ public: } }; -#endif // TIMER_USE_RAW_TICKS +#endif // TIMER_USE_RDTSC // used via TIMER_ACCRUE diff --git a/source/network/SocketBase.cpp b/source/network/SocketBase.cpp index f775469487..eea8834475 100644 --- a/source/network/SocketBase.cpp +++ b/source/network/SocketBase.cpp @@ -76,7 +76,7 @@ CSocketAddress::CSocketAddress(int port, ESocketProtocol proto) break; case IPv6: m_Union.m_IPv6.sin6_family=PF_INET6; - memcpy2(&m_Union.m_IPv6.sin6_addr, &in6addr_any, sizeof(in6addr_any)); + cpu_memcpy(&m_Union.m_IPv6.sin6_addr, &in6addr_any, sizeof(in6addr_any)); m_Union.m_IPv6.sin6_port=htons(port); break; default: @@ -96,7 +96,7 @@ CSocketAddress CSocketAddress::Loopback(int port, ESocketProtocol proto) break; case IPv6: ret.m_Union.m_IPv6.sin6_family=PF_INET6; - memcpy2(&ret.m_Union.m_IPv6.sin6_addr, &in6addr_loopback, sizeof(in6addr_loopback)); + cpu_memcpy(&ret.m_Union.m_IPv6.sin6_addr, &in6addr_loopback, sizeof(in6addr_loopback)); ret.m_Union.m_IPv6.sin6_port=htons(port); break; default: @@ -114,7 +114,7 @@ PS_RESULT CSocketAddress::Resolve(const char *name, int port, CSocketAddress &ad if (res == 0) { if (ai->ai_addrlen < sizeof(addr.m_Union)) - memcpy2(&addr.m_Union, ai->ai_addr, ai->ai_addrlen); + cpu_memcpy(&addr.m_Union, ai->ai_addr, ai->ai_addrlen); switch (addr.m_Union.m_Family) { case IPv4: diff --git a/source/ps/FilePacker.cpp b/source/ps/FilePacker.cpp index 058ad2facd..5760ff615f 100644 --- a/source/ps/FilePacker.cpp +++ b/source/ps/FilePacker.cpp @@ -45,7 +45,7 @@ void CFilePacker::PackRaw(const void* rawdata,u32 rawdatalen) { u32 start=(u32)m_Data.size(); m_Data.resize(m_Data.size()+rawdatalen); - memcpy2(&m_Data[start],rawdata,rawdatalen); + cpu_memcpy(&m_Data[start],rawdata,rawdatalen); *(u32*)&m_Data[8] += rawdatalen; // FIXME byte order? } diff --git a/source/ps/FileUnpacker.cpp b/source/ps/FileUnpacker.cpp index 50a3bfe093..5b77693649 100644 --- a/source/ps/FileUnpacker.cpp +++ b/source/ps/FileUnpacker.cpp @@ -94,7 +94,7 @@ void CFileUnpacker::UnpackRaw(void* rawdata,u32 rawdatalen) { // yes .. copy over void* src = (char*)m_Buf + m_UnpackPos; - memcpy2(rawdata, src, rawdatalen); + cpu_memcpy(rawdata, src, rawdatalen); m_UnpackPos += rawdatalen; } else diff --git a/source/ps/Util.cpp b/source/ps/Util.cpp index abeaaa5edc..d6ac025451 100644 --- a/source/ps/Util.cpp +++ b/source/ps/Util.cpp @@ -48,7 +48,6 @@ void WriteSystemInfo() // get_cpu_info and gfx_detect already called during init - see call site snd_detect(); - get_mem_status(); struct utsname un; uname(&un); @@ -63,9 +62,8 @@ void WriteSystemInfo() fprintf(f, "OS : %s %s (%s)\n", un.sysname, un.release, un.version); // .. CPU - fprintf(f, "CPU : %s, %s", un.machine, cpu_type); - if(cpus > 1) - fprintf(f, " (x%d)", cpus); + fprintf(f, "CPU : %s, %s (%dx%dx%d)", un.machine, cpu_identifierString(), cpu_numPackages(), cpu_coresPerPackage(), cpu_logicalPerPackage()/cpu_coresPerPackage()); + const double cpu_freq = cpu_clockFrequency(); if(cpu_freq != 0.0f) { if(cpu_freq < 1e9) @@ -77,7 +75,7 @@ void WriteSystemInfo() fprintf(f, "\n"); // .. memory - fprintf(f, "Memory : %lu MiB; %lu MiB free\n", tot_mem/MiB, avl_mem/MiB); + fprintf(f, "Memory : %lu MiB; %lu MiB free\n", cpu_memoryTotalMiB(), cpu_memorySize(CPU_MEM_AVAILABLE)/MiB); // .. graphics fprintf(f, "Graphics Card : %s\n", gfx_card); @@ -278,7 +276,7 @@ void WriteBigScreenshot(const char* extension, int tiles) { void* dest = (char*)img + ((tile_y*tile_h + y) * img_w + (tile_x*tile_w)) * bpp/8; void* src = (char*)tile_data + y * tile_w * bpp/8; - memcpy2(dest, src, tile_w * bpp/8); + cpu_memcpy(dest, src, tile_w * bpp/8); } } } diff --git a/source/ps/XML/XMLUtils.cpp b/source/ps/XML/XMLUtils.cpp index 0d5ed8d003..3fe1dd7ee8 100644 --- a/source/ps/XML/XMLUtils.cpp +++ b/source/ps/XML/XMLUtils.cpp @@ -139,7 +139,7 @@ InputSource *CVFSEntityResolver::resolveEntity(const XMLCh *const UNUSED(publicI const ptrdiff_t prefixlen=end-m_DocName; - memcpy2(abspath, m_DocName, prefixlen); + cpu_memcpy(abspath, m_DocName, prefixlen); strncpy(abspath+prefixlen, path, PATH_MAX-prefixlen); // strncpy might not have terminated, if path was too long abspath[PATH_MAX-1]=0; diff --git a/source/ps/XML/Xeromyces.cpp b/source/ps/XML/Xeromyces.cpp index e6f3ed05a7..1f7a9551c1 100644 --- a/source/ps/XML/Xeromyces.cpp +++ b/source/ps/XML/Xeromyces.cpp @@ -41,14 +41,14 @@ public: void write(const void* data, int size) { while (length + size >= allocated) grow(); - memcpy2(&buffer[length], data, size); + cpu_memcpy(&buffer[length], data, size); length += size; } void write(const void* data, int size, int offset) { debug_assert(offset >= 0 && offset+size < length); - memcpy2(&buffer[offset], data, size); + cpu_memcpy(&buffer[offset], data, size); } int tell() diff --git a/source/ps/utf16string.h b/source/ps/utf16string.h index 3cb3448a0b..0284243c2b 100644 --- a/source/ps/utf16string.h +++ b/source/ps/utf16string.h @@ -10,6 +10,8 @@ // native type and distinct from utf16_t. #include +#include "lib/sysdep/cpu.h" + typedef uint16_t utf16_t; typedef std::basic_string utf16string; typedef std::basic_stringstream utf16stringstream; @@ -59,7 +61,7 @@ namespace std { static char_type* copy(char_type* s1, const char_type* s2, size_t n) { - return (char_type *)memcpy2(s1, s2, n*sizeof(char_type)); + return (char_type *)cpu_memcpy(s1, s2, n*sizeof(char_type)); } static char_type* assign(char_type* s, size_t n, char_type a) diff --git a/source/renderer/VertexBuffer.cpp b/source/renderer/VertexBuffer.cpp index a935c29cae..8568f0c524 100644 --- a/source/renderer/VertexBuffer.cpp +++ b/source/renderer/VertexBuffer.cpp @@ -191,7 +191,7 @@ void CVertexBuffer::AppendBatch(VBChunk* UNUSED(chunk),Handle texture,size_t num // resize the chunk's batch to fit its indices batch->m_IndexData.push_back(std::pair(numIndices,indices)); -// memcpy2(&batch->m_Indices[0]+cursize,indices,sizeof(u16)*numIndices); +// cpu_memcpy(&batch->m_Indices[0]+cursize,indices,sizeof(u16)*numIndices); } @@ -207,7 +207,7 @@ void CVertexBuffer::UpdateChunkVertices(VBChunk* chunk,void* data) // if (glGetError() != GL_NO_ERROR) throw PSERROR_Renderer_VBOFailed(); } else { debug_assert(m_SysMem); - memcpy2(m_SysMem+chunk->m_Index*m_VertexSize,data,chunk->m_Count*m_VertexSize); + cpu_memcpy(m_SysMem+chunk->m_Index*m_VertexSize,data,chunk->m_Count*m_VertexSize); } } diff --git a/source/tools/atlas/GameInterface/Handlers/MiscHandlers.cpp b/source/tools/atlas/GameInterface/Handlers/MiscHandlers.cpp index 5d6f98921f..df480d77e2 100644 --- a/source/tools/atlas/GameInterface/Handlers/MiscHandlers.cpp +++ b/source/tools/atlas/GameInterface/Handlers/MiscHandlers.cpp @@ -67,9 +67,9 @@ QUERYHANDLER(CinemaRecord) //* // TODO: BGR24 output doesn't need flipping, YUV420 and RGBA32 do for (int y = 0; y < h/2; ++y) { - memcpy2(temp, &img[y*w*3], w*3); - memcpy2(&img[y*w*3], &img[(h-1-y)*w*3], w*3); - memcpy2(&img[(h-1-y)*w*3], temp, w*3); + cpu_memcpy(temp, &img[y*w*3], w*3); + cpu_memcpy(&img[y*w*3], &img[(h-1-y)*w*3], w*3); + cpu_memcpy(&img[(h-1-y)*w*3], temp, w*3); } //*/