diff --git a/binaries/data/mods/public/simulation/data/players.xml b/binaries/data/mods/public/simulation/data/players.xml index 9d62e14f0a..1fa3e8f7e7 100644 --- a/binaries/data/mods/public/simulation/data/players.xml +++ b/binaries/data/mods/public/simulation/data/players.xml @@ -1,14 +1,14 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/source/lib/sysdep/arch/amd64/amd64_asm.asm b/source/lib/sysdep/arch/amd64/amd64_asm.asm index 3be2422559..87913eca02 100644 --- a/source/lib/sysdep/arch/amd64/amd64_asm.asm +++ b/source/lib/sysdep/arch/amd64/amd64_asm.asm @@ -1,62 +1,62 @@ -; Copyright (c) 2010 Wildfire Games -; -; Permission is hereby granted, free of charge, to any person obtaining -; a copy of this software and associated documentation files (the -; "Software"), to deal in the Software without restriction, including -; without limitation the rights to use, copy, modify, merge, publish, -; distribute, sublicense, and/or sell copies of the Software, and to -; permit persons to whom the Software is furnished to do so, subject to -; the following conditions: -; -; The above copyright notice and this permission notice shall be included -; in all copies or substantial portions of the Software. -; -; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -; EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -; MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -; IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -; CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -; TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -; SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -; For the sym() macro -%include "../ia32/ia32.inc" -; For arg0..3 -%include "amd64_abi.inc" - -BITS 64 - -; extern "C" void __cdecl amd64_asm_cpuid(Ia32CpuidRegs* reg); -; reference: http://softwarecommunity.intel.com/articles/eng/2669.htm -global sym(amd64_asm_cpuid) -sym(amd64_asm_cpuid): - push rbx ; rbx is the only caller-save register we clobber - - mov r8, arg0 - mov eax, DWORD [r8+0] - mov ecx, DWORD [r8+8] - cpuid - mov DWORD [r8+0], eax - mov DWORD [r8+4], ebx - mov DWORD [r8+8], ecx - mov DWORD [r8+12], edx - - pop rbx - - ret - ALIGN 8 - -; extern "C" intptr_t amd64_CAS(volatile uintptr_t *location, uintptr_t expected, uintptr_t newValue); -global sym(amd64_CAS) -sym(amd64_CAS): - mov rax, arg1 ; expected -> rax - lock cmpxchg [arg0], arg2 - sete al - movzx rax, al - ret - -; extern "C" void amd64_AtomicAdd(intptr_t *location, intptr_t increment); -global sym(amd64_AtomicAdd) -sym(amd64_AtomicAdd): - lock add [arg0], arg1 - ret +; Copyright (c) 2010 Wildfire Games +; +; Permission is hereby granted, free of charge, to any person obtaining +; a copy of this software and associated documentation files (the +; "Software"), to deal in the Software without restriction, including +; without limitation the rights to use, copy, modify, merge, publish, +; distribute, sublicense, and/or sell copies of the Software, and to +; permit persons to whom the Software is furnished to do so, subject to +; the following conditions: +; +; The above copyright notice and this permission notice shall be included +; in all copies or substantial portions of the Software. +; +; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +; EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +; MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +; IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +; CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +; TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +; SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +; For the sym() macro +%include "../ia32/ia32.inc" +; For arg0..3 +%include "amd64_abi.inc" + +BITS 64 + +; extern "C" void __cdecl amd64_asm_cpuid(Ia32CpuidRegs* reg); +; reference: http://softwarecommunity.intel.com/articles/eng/2669.htm +global sym(amd64_asm_cpuid) +sym(amd64_asm_cpuid): + push rbx ; rbx is the only caller-save register we clobber + + mov r8, arg0 + mov eax, DWORD [r8+0] + mov ecx, DWORD [r8+8] + cpuid + mov DWORD [r8+0], eax + mov DWORD [r8+4], ebx + mov DWORD [r8+8], ecx + mov DWORD [r8+12], edx + + pop rbx + + ret + ALIGN 8 + +; extern "C" intptr_t amd64_CAS(volatile uintptr_t *location, uintptr_t expected, uintptr_t newValue); +global sym(amd64_CAS) +sym(amd64_CAS): + mov rax, arg1 ; expected -> rax + lock cmpxchg [arg0], arg2 + sete al + movzx rax, al + ret + +; extern "C" void amd64_AtomicAdd(intptr_t *location, intptr_t increment); +global sym(amd64_AtomicAdd) +sym(amd64_AtomicAdd): + lock add [arg0], arg1 + ret diff --git a/source/lib/sysdep/arch/ia32/ia32_asm.asm b/source/lib/sysdep/arch/ia32/ia32_asm.asm index e272e6ae3f..3ba7a6cdb2 100644 --- a/source/lib/sysdep/arch/ia32/ia32_asm.asm +++ b/source/lib/sysdep/arch/ia32/ia32_asm.asm @@ -1,297 +1,297 @@ -; Copyright (c) 2010 Wildfire Games -; -; Permission is hereby granted, free of charge, to any person obtaining -; a copy of this software and associated documentation files (the -; "Software"), to deal in the Software without restriction, including -; without limitation the rights to use, copy, modify, merge, publish, -; distribute, sublicense, and/or sell copies of the Software, and to -; permit persons to whom the Software is furnished to do so, subject to -; the following conditions: -; -; The above copyright notice and this permission notice shall be included -; in all copies or substantial portions of the Software. -; -; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -; EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -; MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -; IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -; CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -; TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -; SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -; optimized assembly code for IA-32. not provided as -; inline assembly because that's compiler-specific. - -%include "ia32.inc" - -; note: pure asm functions prevent inlining but also avoid redundant -; store/loads generated by VC inline asm (ugh). - - -;------------------------------------------------------------------------------- -; CPUID support -;------------------------------------------------------------------------------- - -; extern "C" void __cdecl ia32_asm_cpuid(x86_x64_CpuidRegs* regs); -global sym(ia32_asm_cpuid) -sym(ia32_asm_cpuid): - push ebx ; (clobbered by CPUID) - push edi ; (need a register other than eax..edx) - - mov edi, [esp+8+4] ; -> regs - - mov eax, [edi+0] ; eax (function) - mov ecx, [edi+8] ; ecx (count) - cpuid - stosd - xchg eax, ebx - stosd - xchg eax, ecx - stosd - xchg eax, edx - stosd - - pop edi - pop ebx - ret - - -;------------------------------------------------------------------------------- -; lock-free support routines -;------------------------------------------------------------------------------- - -; extern "C" void __cdecl ia32_asm_AtomicAdd(volatile intptr_t* location, intptr_t increment); -global sym(ia32_asm_AtomicAdd) -sym(ia32_asm_AtomicAdd): - mov edx, [esp+4] ; location - mov eax, [esp+8] ; increment -db 0xf0 ; LOCK prefix - add [edx], eax - ret - - -; notes: -; - a 486 or later processor is required since we use CMPXCHG. -; there's no feature flag we can check, and the ia32 code doesn't -; bother detecting anything < Pentium, so this'll crash and burn if -; run on 386. we could fall back to simple MOVs there (since 386 CPUs -; aren't MP-capable), but it's not worth the trouble. -; - nor do we bother skipping the LOCK prefix on single-processor systems. -; the branch may be well-predicted, but difference in performance still -; isn't expected to be enough to justify the effort. -; extern "C" bool __cdecl ia32_asm_CAS(volatile uintptr_t* location, uintptr_t expected, uintptr_t new_value); -global sym(ia32_asm_CAS) -sym(ia32_asm_CAS): - mov edx, [esp+4] ; location - mov eax, [esp+8] ; expected - mov ecx, [esp+12] ; new_value -db 0xf0 ; LOCK prefix - cmpxchg [edx], ecx - sete al - movzx eax, al - ret - - -;------------------------------------------------------------------------------- -; FPU -;------------------------------------------------------------------------------- - -[section .data] - -; to conform with the fallback implementation (a C cast), we need to -; end up with truncate/"chop" rounding. subtracting does the trick, -; assuming RC is the IA-32 default round-to-nearest mode. -round_bias dd 0.4999999 - -__SECT__ - -; extern "C" u32 __cdecl ia32_asm_control87(u32 new_cw, u32 mask); -global sym(ia32_asm_control87) -sym(ia32_asm_control87): - push eax - fnstcw [esp] - pop eax ; old_cw - mov ecx, [esp+4] ; new_val - mov edx, [esp+8] ; mask - and ecx, edx ; new_val & mask - not edx ; ~mask - and eax, edx ; old_cw & ~mask - or eax, ecx ; (old_cw & ~mask) | (new_val & mask) - push eax ; = new_cw - fldcw [esp] - pop eax - xor eax, eax ; return value - ret - - -; possible IA-32 FPU control word flags after FXAM: NAN|NORMAL|ZERO -FP_CLASSIFY_MASK equ 0x4500 - -; extern "C" size_t __cdecl ia32_asm_fpclassifyd(double d); -global sym(ia32_asm_fpclassifyd) -sym(ia32_asm_fpclassifyd): - fld qword [esp+4] - fxam - fnstsw ax - fstp st0 - and eax, FP_CLASSIFY_MASK - ret - -; extern "C" size_t __cdecl ia32_asm_fpclassifyf(float f); -global sym(ia32_asm_fpclassifyf) -sym(ia32_asm_fpclassifyf): - fld dword [esp+4] - fxam - fnstsw ax - fstp st0 - and eax, FP_CLASSIFY_MASK - ret - - -; extern "C" float __cdecl ia32_asm_rintf(float); -global sym(ia32_asm_rintf) -sym(ia32_asm_rintf): - fld dword [esp+4] - frndint - ret - -; extern "C" double __cdecl ia32_asm_rint(double); -global sym(ia32_asm_rint) -sym(ia32_asm_rint): - fld qword [esp+4] - frndint - ret - - -; extern "C" float __cdecl ia32_asm_fminf(float, float); -global sym(ia32_asm_fminf) -sym(ia32_asm_fminf): - fld dword [esp+4] - fld dword [esp+8] - fcomi st0, st1 - fcmovnb st0, st1 - fxch - fstp st0 - ret - -; extern "C" float __cdecl ia32_asm_fmaxf(float, float); -global sym(ia32_asm_fmaxf) -sym(ia32_asm_fmaxf): - fld dword [esp+4] - fld dword [esp+8] - fcomi st0, st1 - fcmovb st0, st1 - fxch - fstp st0 - ret - - -; extern "C" i32 __cdecl ia32_asm_i32FromFloat(float f); -global sym(ia32_asm_i32FromFloat) -sym(ia32_asm_i32FromFloat): - push eax - fld dword [esp+8] - fsub dword [round_bias] - fistp dword [esp] - pop eax - ret - -; extern "C" i32 __cdecl ia32_asm_i32FromDouble(double d); -global sym(ia32_asm_i32FromDouble) -sym(ia32_asm_i32FromDouble): - push eax - fld qword [esp+8] - fsub dword [round_bias] - fistp dword [esp] - pop eax - ret - -; extern "C" i64 __cdecl ia32_asm_i64FromDouble(double d); -global sym(ia32_asm_i64FromDouble) -sym(ia32_asm_i64FromDouble): - push edx - push eax - fld qword [esp+12] - fsub dword [round_bias] - fistp qword [esp] - pop eax - pop edx - ret - - -;------------------------------------------------------------------------------- -; misc -;------------------------------------------------------------------------------- - -; write the current execution state (e.g. all register values) into -; (Win32::CONTEXT*)pcontext (defined as void* to avoid dependency). -; optimized for size; this must be straight asm because ; extern "C" -; is compiler-specific and compiler-generated prolog code inserted before -; inline asm trashes EBP and ESP (unacceptable). -; extern "C" void ia32_asm_GetCurrentContext(void* pcontext); -global sym(ia32_asm_GetCurrentContext) -sym(ia32_asm_GetCurrentContext): - pushad - pushfd - mov edi, [esp+4+32+4] ; pcontext - - ; ContextFlags - mov eax, 0x10007 ; segs, int, control - stosd - - ; DRx and FloatSave - ; rationale: we can't access the debug registers from Ring3, and - ; the FPU save area is irrelevant, so zero them. - xor eax, eax - push byte 6+8+20 - pop ecx -rep stosd - - ; CONTEXT_SEGMENTS - mov ax, gs - stosd - mov ax, fs - stosd - mov ax, es - stosd - mov ax, ds - stosd - - ; CONTEXT_INTEGER - mov eax, [esp+4+32-32] ; edi - stosd - xchg eax, esi - stosd - xchg eax, ebx - stosd - xchg eax, edx - stosd - mov eax, [esp+4+32-8] ; ecx - stosd - mov eax, [esp+4+32-4] ; eax - stosd - - ; CONTEXT_CONTROL - xchg eax, ebp ; ebp restored by POPAD - stosd - mov eax, [esp+4+32] ; return address - sub eax, 5 ; skip CALL instruction -> call site. - stosd - xor eax, eax - mov ax, cs - stosd - pop eax ; eflags - stosd - lea eax, [esp+32+4+4] ; esp - stosd - xor eax, eax - mov ax, ss - stosd - - ; ExtendedRegisters - xor ecx, ecx - mov cl, 512/4 -rep stosd - - popad - ret +; Copyright (c) 2010 Wildfire Games +; +; Permission is hereby granted, free of charge, to any person obtaining +; a copy of this software and associated documentation files (the +; "Software"), to deal in the Software without restriction, including +; without limitation the rights to use, copy, modify, merge, publish, +; distribute, sublicense, and/or sell copies of the Software, and to +; permit persons to whom the Software is furnished to do so, subject to +; the following conditions: +; +; The above copyright notice and this permission notice shall be included +; in all copies or substantial portions of the Software. +; +; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +; EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +; MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +; IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +; CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +; TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +; SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +; optimized assembly code for IA-32. not provided as +; inline assembly because that's compiler-specific. + +%include "ia32.inc" + +; note: pure asm functions prevent inlining but also avoid redundant +; store/loads generated by VC inline asm (ugh). + + +;------------------------------------------------------------------------------- +; CPUID support +;------------------------------------------------------------------------------- + +; extern "C" void __cdecl ia32_asm_cpuid(x86_x64_CpuidRegs* regs); +global sym(ia32_asm_cpuid) +sym(ia32_asm_cpuid): + push ebx ; (clobbered by CPUID) + push edi ; (need a register other than eax..edx) + + mov edi, [esp+8+4] ; -> regs + + mov eax, [edi+0] ; eax (function) + mov ecx, [edi+8] ; ecx (count) + cpuid + stosd + xchg eax, ebx + stosd + xchg eax, ecx + stosd + xchg eax, edx + stosd + + pop edi + pop ebx + ret + + +;------------------------------------------------------------------------------- +; lock-free support routines +;------------------------------------------------------------------------------- + +; extern "C" void __cdecl ia32_asm_AtomicAdd(volatile intptr_t* location, intptr_t increment); +global sym(ia32_asm_AtomicAdd) +sym(ia32_asm_AtomicAdd): + mov edx, [esp+4] ; location + mov eax, [esp+8] ; increment +db 0xf0 ; LOCK prefix + add [edx], eax + ret + + +; notes: +; - a 486 or later processor is required since we use CMPXCHG. +; there's no feature flag we can check, and the ia32 code doesn't +; bother detecting anything < Pentium, so this'll crash and burn if +; run on 386. we could fall back to simple MOVs there (since 386 CPUs +; aren't MP-capable), but it's not worth the trouble. +; - nor do we bother skipping the LOCK prefix on single-processor systems. +; the branch may be well-predicted, but difference in performance still +; isn't expected to be enough to justify the effort. +; extern "C" bool __cdecl ia32_asm_CAS(volatile uintptr_t* location, uintptr_t expected, uintptr_t new_value); +global sym(ia32_asm_CAS) +sym(ia32_asm_CAS): + mov edx, [esp+4] ; location + mov eax, [esp+8] ; expected + mov ecx, [esp+12] ; new_value +db 0xf0 ; LOCK prefix + cmpxchg [edx], ecx + sete al + movzx eax, al + ret + + +;------------------------------------------------------------------------------- +; FPU +;------------------------------------------------------------------------------- + +[section .data] + +; to conform with the fallback implementation (a C cast), we need to +; end up with truncate/"chop" rounding. subtracting does the trick, +; assuming RC is the IA-32 default round-to-nearest mode. +round_bias dd 0.4999999 + +__SECT__ + +; extern "C" u32 __cdecl ia32_asm_control87(u32 new_cw, u32 mask); +global sym(ia32_asm_control87) +sym(ia32_asm_control87): + push eax + fnstcw [esp] + pop eax ; old_cw + mov ecx, [esp+4] ; new_val + mov edx, [esp+8] ; mask + and ecx, edx ; new_val & mask + not edx ; ~mask + and eax, edx ; old_cw & ~mask + or eax, ecx ; (old_cw & ~mask) | (new_val & mask) + push eax ; = new_cw + fldcw [esp] + pop eax + xor eax, eax ; return value + ret + + +; possible IA-32 FPU control word flags after FXAM: NAN|NORMAL|ZERO +FP_CLASSIFY_MASK equ 0x4500 + +; extern "C" size_t __cdecl ia32_asm_fpclassifyd(double d); +global sym(ia32_asm_fpclassifyd) +sym(ia32_asm_fpclassifyd): + fld qword [esp+4] + fxam + fnstsw ax + fstp st0 + and eax, FP_CLASSIFY_MASK + ret + +; extern "C" size_t __cdecl ia32_asm_fpclassifyf(float f); +global sym(ia32_asm_fpclassifyf) +sym(ia32_asm_fpclassifyf): + fld dword [esp+4] + fxam + fnstsw ax + fstp st0 + and eax, FP_CLASSIFY_MASK + ret + + +; extern "C" float __cdecl ia32_asm_rintf(float); +global sym(ia32_asm_rintf) +sym(ia32_asm_rintf): + fld dword [esp+4] + frndint + ret + +; extern "C" double __cdecl ia32_asm_rint(double); +global sym(ia32_asm_rint) +sym(ia32_asm_rint): + fld qword [esp+4] + frndint + ret + + +; extern "C" float __cdecl ia32_asm_fminf(float, float); +global sym(ia32_asm_fminf) +sym(ia32_asm_fminf): + fld dword [esp+4] + fld dword [esp+8] + fcomi st0, st1 + fcmovnb st0, st1 + fxch + fstp st0 + ret + +; extern "C" float __cdecl ia32_asm_fmaxf(float, float); +global sym(ia32_asm_fmaxf) +sym(ia32_asm_fmaxf): + fld dword [esp+4] + fld dword [esp+8] + fcomi st0, st1 + fcmovb st0, st1 + fxch + fstp st0 + ret + + +; extern "C" i32 __cdecl ia32_asm_i32FromFloat(float f); +global sym(ia32_asm_i32FromFloat) +sym(ia32_asm_i32FromFloat): + push eax + fld dword [esp+8] + fsub dword [round_bias] + fistp dword [esp] + pop eax + ret + +; extern "C" i32 __cdecl ia32_asm_i32FromDouble(double d); +global sym(ia32_asm_i32FromDouble) +sym(ia32_asm_i32FromDouble): + push eax + fld qword [esp+8] + fsub dword [round_bias] + fistp dword [esp] + pop eax + ret + +; extern "C" i64 __cdecl ia32_asm_i64FromDouble(double d); +global sym(ia32_asm_i64FromDouble) +sym(ia32_asm_i64FromDouble): + push edx + push eax + fld qword [esp+12] + fsub dword [round_bias] + fistp qword [esp] + pop eax + pop edx + ret + + +;------------------------------------------------------------------------------- +; misc +;------------------------------------------------------------------------------- + +; write the current execution state (e.g. all register values) into +; (Win32::CONTEXT*)pcontext (defined as void* to avoid dependency). +; optimized for size; this must be straight asm because ; extern "C" +; is compiler-specific and compiler-generated prolog code inserted before +; inline asm trashes EBP and ESP (unacceptable). +; extern "C" void ia32_asm_GetCurrentContext(void* pcontext); +global sym(ia32_asm_GetCurrentContext) +sym(ia32_asm_GetCurrentContext): + pushad + pushfd + mov edi, [esp+4+32+4] ; pcontext + + ; ContextFlags + mov eax, 0x10007 ; segs, int, control + stosd + + ; DRx and FloatSave + ; rationale: we can't access the debug registers from Ring3, and + ; the FPU save area is irrelevant, so zero them. + xor eax, eax + push byte 6+8+20 + pop ecx +rep stosd + + ; CONTEXT_SEGMENTS + mov ax, gs + stosd + mov ax, fs + stosd + mov ax, es + stosd + mov ax, ds + stosd + + ; CONTEXT_INTEGER + mov eax, [esp+4+32-32] ; edi + stosd + xchg eax, esi + stosd + xchg eax, ebx + stosd + xchg eax, edx + stosd + mov eax, [esp+4+32-8] ; ecx + stosd + mov eax, [esp+4+32-4] ; eax + stosd + + ; CONTEXT_CONTROL + xchg eax, ebp ; ebp restored by POPAD + stosd + mov eax, [esp+4+32] ; return address + sub eax, 5 ; skip CALL instruction -> call site. + stosd + xor eax, eax + mov ax, cs + stosd + pop eax ; eflags + stosd + lea eax, [esp+32+4+4] ; esp + stosd + xor eax, eax + mov ax, ss + stosd + + ; ExtendedRegisters + xor ecx, ecx + mov cl, 512/4 +rep stosd + + popad + ret diff --git a/source/lib/utf8.cpp b/source/lib/utf8.cpp index 917f0fd8f8..6d2d49225a 100644 --- a/source/lib/utf8.cpp +++ b/source/lib/utf8.cpp @@ -1,237 +1,237 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "precompiled.h" -#include "lib/utf8.h" - -ERROR_ASSOCIATE(ERR::UTF8_SURROGATE, L"UTF-16 surrogate pairs aren't supported", -1); -ERROR_ASSOCIATE(ERR::UTF8_OUTSIDE_BMP, L"Code point outside BMP (> 0x10000)", -1); -ERROR_ASSOCIATE(ERR::UTF8_NONCHARACTER, L"Noncharacter (e.g. WEOF)", -1); -ERROR_ASSOCIATE(ERR::UTF8_INVALID_UTF8, L"Invalid UTF-8 sequence", -1); - - -// adapted from http://unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c -// which bears the following notice: -/* -* Copyright 2001-2004 Unicode, Inc. -* -* Disclaimer -* -* This source code is provided as is by Unicode, Inc. No claims are -* made as to fitness for any particular purpose. No warranties of any -* kind are expressed or implied. The recipient agrees to determine -* applicability of information provided. If this file has been -* purchased on magnetic or optical media from Unicode, Inc., the -* sole remedy for any claim will be exchange of defective media -* within 90 days of receipt. -* -* Limitations on Rights to Redistribute This Code -* -* Unicode, Inc. hereby grants the right to freely use the information -* supplied in this file in the creation of products supporting the -* Unicode Standard, and to make copies of this file in any form -* for internal or external distribution as long as this notice -* remains attached. -*/ - -// design rationale: -// - to cope with wchar_t differences between VC (UTF-16) and -// GCC (UCS-4), we only allow codepoints in the BMP. -// encoded UTF-8 sequences are therefore no longer than 3 bytes. -// - surrogates are disabled because variable-length strings -// violate the purpose of using wchar_t instead of UTF-8. -// - replacing disallowed characters instead of aborting outright -// avoids overly inconveniencing users and eases debugging. - -// this implementation survives http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt - -// (must be unsigned to avoid sign extension) -typedef u8 UTF8; -typedef u32 UTF32; - - -// called from ReplaceIfInvalid and UTF8Codec::Decode -static UTF32 RaiseError(LibError err, LibError* perr) -{ - if(perr) // caller wants return code, not warning dialog - { - if(*perr == INFO::OK) // only return the first error (see header) - *perr = err; - } - else - DEBUG_WARN_ERR(err); - - return 0xFFFDul; // replacement character -} - - -static UTF32 ReplaceIfInvalid(UTF32 u, LibError* err) -{ - // disallow surrogates - if(0xD800ul <= u && u <= 0xDFFFul) - return RaiseError(ERR::UTF8_SURROGATE, err); - // outside BMP (UTF-16 representation would require surrogates) - if(u > 0xFFFFul) - return RaiseError(ERR::UTF8_OUTSIDE_BMP, err); - // noncharacter (note: WEOF (0xFFFF) causes VC's swprintf to fail) - if(u == 0xFFFEul || u == 0xFFFFul || (0xFDD0ul <= u && u <= 0xFDEFul)) - return RaiseError(ERR::UTF8_NONCHARACTER, err); - return u; -} - - -class UTF8Codec -{ -public: - static void Encode(UTF32 u, UTF8*& dstPos) - { - switch (Size(u)) - { - case 1: - *dstPos++ = UTF8(u); - break; - case 2: - *dstPos++ = UTF8((u >> 6) | 0xC0); - *dstPos++ = UTF8((u | 0x80u) & 0xBFu); - break; - case 3: - *dstPos++ = UTF8((u >> 12) | 0xE0); - *dstPos++ = UTF8(((u >> 6) | 0x80u) & 0xBFu); - *dstPos++ = UTF8((u | 0x80u) & 0xBFu); - break; - } - } - - // @return decoded scalar, or replacementCharacter on error - static UTF32 Decode(const UTF8*& srcPos, const UTF8* const srcEnd, LibError* err) - { - const size_t size = SizeFromFirstByte(*srcPos); - if(!IsValid(srcPos, size, srcEnd)) - { - srcPos += 1; // only skip the offending byte (increases chances of resynchronization) - return RaiseError(ERR::UTF8_INVALID_UTF8, err); - } - - UTF32 u = 0; - for(size_t i = 0; i < size-1; i++) - { - u += UTF32(*srcPos++); - u <<= 6; - } - u += UTF32(*srcPos++); - - static const UTF32 offsets[1+4] = { 0, 0x00000000ul, 0x00003080ul, 0x000E2080ul, 0x03C82080UL }; - u -= offsets[size]; - return u; - } - -private: - static inline size_t Size(UTF32 u) - { - if(u < 0x80) - return 1; - if(u < 0x800) - return 2; - // ReplaceIfInvalid ensures > 3 byte encodings are never used. - return 3; - } - - static inline size_t SizeFromFirstByte(UTF8 firstByte) - { - if(firstByte < 0xC0) - return 1; - if(firstByte < 0xE0) - return 2; - if(firstByte < 0xF0) - return 3; - // IsValid rejects firstByte values that would cause > 4 byte encodings. - return 4; - } - - // c.f. Unicode 3.1 Table 3-7 - // @param size obtained via SizeFromFirstByte (our caller also uses it) - static bool IsValid(const UTF8* const src, size_t size, const UTF8* const srcEnd) - { - if(src+size > srcEnd) // not enough data - return false; - - if(src[0] < 0x80) - return true; - if(!(0xC2 <= src[0] && src[0] <= 0xF4)) - return false; - - // special cases (stricter than the loop) - if(src[0] == 0xE0 && src[1] < 0xA0) - return false; - if(src[0] == 0xED && src[1] > 0x9F) - return false; - if(src[0] == 0xF0 && src[1] < 0x90) - return false; - if(src[0] == 0xF4 && src[1] > 0x8F) - return false; - - for(size_t i = 1; i < size; i++) - { - if(!(0x80 <= src[i] && src[i] <= 0xBF)) - return false; - } - - return true; - } -}; - - -//----------------------------------------------------------------------------- - -std::string utf8_from_wstring(const std::wstring& src, LibError* err) -{ - if(err) - *err = INFO::OK; - - std::string dst(src.size()*3+1, ' '); // see UTF8Codec::Size; +1 ensures &dst[0] is valid - UTF8* dstPos = (UTF8*)&dst[0]; - for(size_t i = 0; i < src.size(); i++) - { - const UTF32 u = ReplaceIfInvalid(UTF32(src[i]), err); - UTF8Codec::Encode(u, dstPos); - } - dst.resize(dstPos - (UTF8*)&dst[0]); - return dst; -} - - -std::wstring wstring_from_utf8(const std::string& src, LibError* err) -{ - if(err) - *err = INFO::OK; - - std::wstring dst; - dst.reserve(src.size()); - const UTF8* srcPos = (const UTF8*)src.data(); - const UTF8* const srcEnd = srcPos + src.size(); - while(srcPos < srcEnd) - { - const UTF32 u = UTF8Codec::Decode(srcPos, srcEnd, err); - dst.push_back((wchar_t)ReplaceIfInvalid(u, err)); - } - return dst; -} +/* Copyright (c) 2010 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "precompiled.h" +#include "lib/utf8.h" + +ERROR_ASSOCIATE(ERR::UTF8_SURROGATE, L"UTF-16 surrogate pairs aren't supported", -1); +ERROR_ASSOCIATE(ERR::UTF8_OUTSIDE_BMP, L"Code point outside BMP (> 0x10000)", -1); +ERROR_ASSOCIATE(ERR::UTF8_NONCHARACTER, L"Noncharacter (e.g. WEOF)", -1); +ERROR_ASSOCIATE(ERR::UTF8_INVALID_UTF8, L"Invalid UTF-8 sequence", -1); + + +// adapted from http://unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c +// which bears the following notice: +/* +* Copyright 2001-2004 Unicode, Inc. +* +* Disclaimer +* +* This source code is provided as is by Unicode, Inc. No claims are +* made as to fitness for any particular purpose. No warranties of any +* kind are expressed or implied. The recipient agrees to determine +* applicability of information provided. If this file has been +* purchased on magnetic or optical media from Unicode, Inc., the +* sole remedy for any claim will be exchange of defective media +* within 90 days of receipt. +* +* Limitations on Rights to Redistribute This Code +* +* Unicode, Inc. hereby grants the right to freely use the information +* supplied in this file in the creation of products supporting the +* Unicode Standard, and to make copies of this file in any form +* for internal or external distribution as long as this notice +* remains attached. +*/ + +// design rationale: +// - to cope with wchar_t differences between VC (UTF-16) and +// GCC (UCS-4), we only allow codepoints in the BMP. +// encoded UTF-8 sequences are therefore no longer than 3 bytes. +// - surrogates are disabled because variable-length strings +// violate the purpose of using wchar_t instead of UTF-8. +// - replacing disallowed characters instead of aborting outright +// avoids overly inconveniencing users and eases debugging. + +// this implementation survives http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + +// (must be unsigned to avoid sign extension) +typedef u8 UTF8; +typedef u32 UTF32; + + +// called from ReplaceIfInvalid and UTF8Codec::Decode +static UTF32 RaiseError(LibError err, LibError* perr) +{ + if(perr) // caller wants return code, not warning dialog + { + if(*perr == INFO::OK) // only return the first error (see header) + *perr = err; + } + else + DEBUG_WARN_ERR(err); + + return 0xFFFDul; // replacement character +} + + +static UTF32 ReplaceIfInvalid(UTF32 u, LibError* err) +{ + // disallow surrogates + if(0xD800ul <= u && u <= 0xDFFFul) + return RaiseError(ERR::UTF8_SURROGATE, err); + // outside BMP (UTF-16 representation would require surrogates) + if(u > 0xFFFFul) + return RaiseError(ERR::UTF8_OUTSIDE_BMP, err); + // noncharacter (note: WEOF (0xFFFF) causes VC's swprintf to fail) + if(u == 0xFFFEul || u == 0xFFFFul || (0xFDD0ul <= u && u <= 0xFDEFul)) + return RaiseError(ERR::UTF8_NONCHARACTER, err); + return u; +} + + +class UTF8Codec +{ +public: + static void Encode(UTF32 u, UTF8*& dstPos) + { + switch (Size(u)) + { + case 1: + *dstPos++ = UTF8(u); + break; + case 2: + *dstPos++ = UTF8((u >> 6) | 0xC0); + *dstPos++ = UTF8((u | 0x80u) & 0xBFu); + break; + case 3: + *dstPos++ = UTF8((u >> 12) | 0xE0); + *dstPos++ = UTF8(((u >> 6) | 0x80u) & 0xBFu); + *dstPos++ = UTF8((u | 0x80u) & 0xBFu); + break; + } + } + + // @return decoded scalar, or replacementCharacter on error + static UTF32 Decode(const UTF8*& srcPos, const UTF8* const srcEnd, LibError* err) + { + const size_t size = SizeFromFirstByte(*srcPos); + if(!IsValid(srcPos, size, srcEnd)) + { + srcPos += 1; // only skip the offending byte (increases chances of resynchronization) + return RaiseError(ERR::UTF8_INVALID_UTF8, err); + } + + UTF32 u = 0; + for(size_t i = 0; i < size-1; i++) + { + u += UTF32(*srcPos++); + u <<= 6; + } + u += UTF32(*srcPos++); + + static const UTF32 offsets[1+4] = { 0, 0x00000000ul, 0x00003080ul, 0x000E2080ul, 0x03C82080UL }; + u -= offsets[size]; + return u; + } + +private: + static inline size_t Size(UTF32 u) + { + if(u < 0x80) + return 1; + if(u < 0x800) + return 2; + // ReplaceIfInvalid ensures > 3 byte encodings are never used. + return 3; + } + + static inline size_t SizeFromFirstByte(UTF8 firstByte) + { + if(firstByte < 0xC0) + return 1; + if(firstByte < 0xE0) + return 2; + if(firstByte < 0xF0) + return 3; + // IsValid rejects firstByte values that would cause > 4 byte encodings. + return 4; + } + + // c.f. Unicode 3.1 Table 3-7 + // @param size obtained via SizeFromFirstByte (our caller also uses it) + static bool IsValid(const UTF8* const src, size_t size, const UTF8* const srcEnd) + { + if(src+size > srcEnd) // not enough data + return false; + + if(src[0] < 0x80) + return true; + if(!(0xC2 <= src[0] && src[0] <= 0xF4)) + return false; + + // special cases (stricter than the loop) + if(src[0] == 0xE0 && src[1] < 0xA0) + return false; + if(src[0] == 0xED && src[1] > 0x9F) + return false; + if(src[0] == 0xF0 && src[1] < 0x90) + return false; + if(src[0] == 0xF4 && src[1] > 0x8F) + return false; + + for(size_t i = 1; i < size; i++) + { + if(!(0x80 <= src[i] && src[i] <= 0xBF)) + return false; + } + + return true; + } +}; + + +//----------------------------------------------------------------------------- + +std::string utf8_from_wstring(const std::wstring& src, LibError* err) +{ + if(err) + *err = INFO::OK; + + std::string dst(src.size()*3+1, ' '); // see UTF8Codec::Size; +1 ensures &dst[0] is valid + UTF8* dstPos = (UTF8*)&dst[0]; + for(size_t i = 0; i < src.size(); i++) + { + const UTF32 u = ReplaceIfInvalid(UTF32(src[i]), err); + UTF8Codec::Encode(u, dstPos); + } + dst.resize(dstPos - (UTF8*)&dst[0]); + return dst; +} + + +std::wstring wstring_from_utf8(const std::string& src, LibError* err) +{ + if(err) + *err = INFO::OK; + + std::wstring dst; + dst.reserve(src.size()); + const UTF8* srcPos = (const UTF8*)src.data(); + const UTF8* const srcEnd = srcPos + src.size(); + while(srcPos < srcEnd) + { + const UTF32 u = UTF8Codec::Decode(srcPos, srcEnd, err); + dst.push_back((wchar_t)ReplaceIfInvalid(u, err)); + } + return dst; +} diff --git a/source/lib/utf8.h b/source/lib/utf8.h index e85336fdf0..c47f9b2496 100644 --- a/source/lib/utf8.h +++ b/source/lib/utf8.h @@ -1,52 +1,52 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef INCLUDED_UTF8 -#define INCLUDED_UTF8 - -// note: error codes are returned via optional output parameter. -namespace ERR -{ - const LibError UTF8_SURROGATE = -100700; - const LibError UTF8_OUTSIDE_BMP = -100701; - const LibError UTF8_NONCHARACTER = -100702; - const LibError UTF8_INVALID_UTF8 = -100703; -} - -/** - * convert UTF-8 to a wide string (UTF-16 or UCS-4, depending on the - * platform's wchar_t). - * - * @param s input string (UTF-8) - * @param err if nonzero, this receives the first error encountered - * (the rest may be subsequent faults) or INFO::OK if all went well. - * otherwise, the function raises a warning dialog for every - * error/warning. - **/ -LIB_API std::wstring wstring_from_utf8(const std::string& s, LibError* err = 0); - -/** - * opposite of wstring_from_utf8 - **/ -LIB_API std::string utf8_from_wstring(const std::wstring& s, LibError* err = 0); - -#endif // #ifndef INCLUDED_UTF8 +/* Copyright (c) 2010 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef INCLUDED_UTF8 +#define INCLUDED_UTF8 + +// note: error codes are returned via optional output parameter. +namespace ERR +{ + const LibError UTF8_SURROGATE = -100700; + const LibError UTF8_OUTSIDE_BMP = -100701; + const LibError UTF8_NONCHARACTER = -100702; + const LibError UTF8_INVALID_UTF8 = -100703; +} + +/** + * convert UTF-8 to a wide string (UTF-16 or UCS-4, depending on the + * platform's wchar_t). + * + * @param s input string (UTF-8) + * @param err if nonzero, this receives the first error encountered + * (the rest may be subsequent faults) or INFO::OK if all went well. + * otherwise, the function raises a warning dialog for every + * error/warning. + **/ +LIB_API std::wstring wstring_from_utf8(const std::string& s, LibError* err = 0); + +/** + * opposite of wstring_from_utf8 + **/ +LIB_API std::string utf8_from_wstring(const std::wstring& s, LibError* err = 0); + +#endif // #ifndef INCLUDED_UTF8