diff --git a/source/gui/CGUI.cpp b/source/gui/CGUI.cpp index 076818fea0..1b288f1ea8 100644 --- a/source/gui/CGUI.cpp +++ b/source/gui/CGUI.cpp @@ -1557,7 +1557,7 @@ void CGUI::Xeromyces_ReadImage(XMBElement Element, CXeromyces* pFile, CGUISprite if (attr_name == "texture") { - image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_UTF8(attr_value); + image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_utf8(attr_value); } else if (attr_name == "size") diff --git a/source/gui/GUIRenderer.cpp b/source/gui/GUIRenderer.cpp index 2994d5e32e..49d2f8481f 100644 --- a/source/gui/GUIRenderer.cpp +++ b/source/gui/GUIRenderer.cpp @@ -370,7 +370,7 @@ void GUIRenderer::UpdateDrawCallCache(DrawCalls &Calls, const CStr& SpriteName, if (SpriteName.substr(0, 10) == "stretched:") { SGUIImage Image; - Image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_UTF8(SpriteName.substr(10)); + Image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_utf8(SpriteName.substr(10)); CClientArea ca("0 0 100% 100%"); Image.m_Size = ca; Image.m_TextureSize = ca; diff --git a/source/lib/file/archive/archive_zip.cpp b/source/lib/file/archive/archive_zip.cpp index 933252aa2c..92a859456b 100644 --- a/source/lib/file/archive/archive_zip.cpp +++ b/source/lib/file/archive/archive_zip.cpp @@ -27,7 +27,7 @@ #include "lib/bits.h" #include "lib/byte_order.h" -#include "lib/wchar.h" // wstring_from_UTF8 +#include "lib/wchar.h" // wstring_from_utf8 #include "lib/fat_time.h" #include "lib/path_util.h" #include "lib/allocators/pool.h" @@ -135,7 +135,7 @@ public: { const size_t length = (size_t)read_le16(&m_fn_len); const char* pathname = (const char*)this + sizeof(CDFH); // not 0-terminated! - return wstring_from_UTF8(std::string(pathname, length)); + return wstring_from_utf8(std::string(pathname, length)); } off_t HeaderOffset() const diff --git a/source/lib/file/file_system.cpp b/source/lib/file/file_system.cpp index 67dd80336b..4047f69bd6 100644 --- a/source/lib/file/file_system.cpp +++ b/source/lib/file/file_system.cpp @@ -23,7 +23,7 @@ #include #include "lib/path_util.h" -#include "lib/wchar.h" // wstring_from_UTF8 +#include "lib/wchar.h" // wstring_from_utf8 #include "lib/posix/posix_filesystem.h" @@ -67,7 +67,7 @@ LibError GetDirectoryEntries(const fs::wpath& path, FileInfos* files, DirectoryN return LibError_from_errno(); } - const std::wstring name = wstring_from_UTF8(osEnt->d_name); + const std::wstring name = wstring_from_utf8(osEnt->d_name); RETURN_ERR(path_component_validate(name.c_str())); // get file information (mode, size, mtime) diff --git a/source/lib/path_util.cpp b/source/lib/path_util.cpp index 4e00121616..f5ff454e25 100644 --- a/source/lib/path_util.cpp +++ b/source/lib/path_util.cpp @@ -128,10 +128,10 @@ const wchar_t* path_name_only(const wchar_t* path) fs::wpath wpath_from_path(const fs::path& pathname) { - return wstring_from_UTF8(pathname.string()); + return wstring_from_utf8(pathname.string()); } fs::path path_from_wpath(const fs::wpath& pathname) { - return UTF8_from_wstring(pathname.string()); + return utf8_from_wstring(pathname.string()); } diff --git a/source/lib/secure_crt.cpp b/source/lib/secure_crt.cpp index 27d8447c6b..40708490ee 100644 --- a/source/lib/secure_crt.cpp +++ b/source/lib/secure_crt.cpp @@ -257,8 +257,8 @@ int tsprintf_s(tchar* buf, size_t max_chars, const tchar* fmt, ...) errno_t _wfopen_s(FILE** pfile, const wchar_t* filename, const wchar_t* mode) { *pfile = NULL; - const std::string filename_c = UTF8_from_wstring(filename); - const std::string mode_c = UTF8_from_wstring(mode); + const std::string filename_c = utf8_from_wstring(filename); + const std::string mode_c = utf8_from_wstring(mode); return fopen_s(pfile, filename_c.c_str(), mode_c.c_str()); } diff --git a/source/lib/sysdep/os/win/wseh.cpp b/source/lib/sysdep/os/win/wseh.cpp index 31c5999cac..4c008360cd 100644 --- a/source/lib/sysdep/os/win/wseh.cpp +++ b/source/lib/sysdep/os/win/wseh.cpp @@ -281,7 +281,7 @@ long __stdcall wseh_ExceptionFilter(struct _EXCEPTION_POINTERS* ep) if(ep->ExceptionRecord->ExceptionFlags & EXCEPTION_NONCONTINUABLE) flags = DE_NO_CONTINUE; const wchar_t* const lastFuncToSkip = WIDEN(STRINGIZE(DECORATED_NAME(wseh_ExceptionFilter))); - ErrorReaction er = debug_DisplayError(message, flags, ep->ContextRecord, lastFuncToSkip, file,line,UTF8_from_wstring(func).c_str(), 0); + ErrorReaction er = debug_DisplayError(message, flags, ep->ContextRecord, lastFuncToSkip, file,line,utf8_from_wstring(func).c_str(), 0); debug_assert(er == ER_CONTINUE); // nothing else possible // invoke the Win32 default handler - it calls ExitProcess for diff --git a/source/lib/sysdep/tests/test_sysdep.h b/source/lib/sysdep/tests/test_sysdep.h index a4d3518362..44f3d24640 100644 --- a/source/lib/sysdep/tests/test_sysdep.h +++ b/source/lib/sysdep/tests/test_sysdep.h @@ -108,7 +108,7 @@ public: sprintf_s(root, ARRAY_SIZE(root), "%s/pyrogenesis-test-sysdep-XXXXXX", tmpdir); TS_ASSERT(mkdtemp(root)); std::string rootstr(root); - std::wstring rootstrw(wstring_from_UTF8(rootstr)); + std::wstring rootstrw(wstring_from_utf8(rootstr)); const char* dirs[] = { "/example", diff --git a/source/lib/tests/test_wchar.h b/source/lib/tests/test_wchar.h new file mode 100644 index 0000000000..fe254af9f6 --- /dev/null +++ b/source/lib/tests/test_wchar.h @@ -0,0 +1,78 @@ +/* Copyright (C) 2009 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#include "lib/self_test.h" +#include "lib/wchar.h" + +// (copied from CStr test) + +class Test_wchar : public CxxTest::TestSuite +{ +public: + void test_utf8_utf16_conversion() + { + const wchar_t chr_utf16[] = { + 0x12, + 0xff, + 0x1234, + 0x3456, + 0x5678, + 0x7890, + 0x9abc, + 0xbcde, + 0xfffd + }; + const unsigned char chr_utf8[] = { + 0x12, + 0xc3, 0xbf, + 0xe1, 0x88, 0xb4, + 0xe3, 0x91, 0x96, + 0xe5, 0x99, 0xb8, + 0xe7, 0xa2, 0x90, + 0xe9, 0xaa, 0xbc, + 0xeb, 0xb3, 0x9e, + 0xef, 0xbf, 0xbd + }; + const std::wstring str_utf16(chr_utf16, ARRAY_SIZE(chr_utf16)); + + const std::string str_utf8 = utf8_from_wstring(str_utf16); + TS_ASSERT_EQUALS(str_utf8.length(), ARRAY_SIZE(chr_utf8)); + TS_ASSERT_SAME_DATA(str_utf8.data(), chr_utf8, ARRAY_SIZE(chr_utf8)*sizeof(char)); + + const std::wstring str_utf16b = wstring_from_utf8(str_utf8); + TS_ASSERT_WSTR_EQUALS(str_utf16b, str_utf16); + } + + void test_invalid_utf8() + { + struct { const char* utf8; const wchar_t* utf16; } tests[] = { + { "a\xef", L"a\xfffd" }, + { "b\xef\xbf", L"b\xfffd\xfffd" }, + { "c\xef\xbf\x01", L"c\xfffd\xfffd\x0001" }, + { "d\xffX\x80Y\x80" , L"d\xfffdX\xfffdY\xfffd" } + }; + for (size_t i = 0; i < ARRAY_SIZE(tests); ++i) + { + const std::string str_utf8(tests[i].utf8); + const std::wstring str_utf16(tests[i].utf16); + + const std::wstring str_utf8to16 = wstring_from_utf8(str_utf8); + TS_ASSERT_EQUALS(str_utf16.length(), str_utf8to16.length()); + TS_ASSERT_SAME_DATA(str_utf8to16.data(), str_utf16.data(), str_utf16.length()*sizeof(wchar_t)); + } + } +}; diff --git a/source/lib/wchar.cpp b/source/lib/wchar.cpp index a9ec036331..323c9d9d33 100644 --- a/source/lib/wchar.cpp +++ b/source/lib/wchar.cpp @@ -55,25 +55,20 @@ typedef u8 UTF8; typedef u32 UTF32; +static const UTF32 replacementCharacter = 0xFFFDul; // used by ReplaceIfInvalid and UTF8::Decode static UTF32 ReplaceIfInvalid(UTF32 u) { - struct IsValid - { - bool operator()(UTF32 u) const - { - // disallow surrogates - if(0xDC00ul <= u && u <= 0xDFFFul) - return false; - // greater: UTF-16 representation would require surrogates - // equal: permanently unassigned codepoint, may correspond to WEOF - // (which raises errors when used in VC's swprintf) - if(u >= 0xFFFFul) - return false; - return true; - } - }; - return IsValid()(u)? u : 0xFFFD; + // disallow surrogates + if(0xD800ul <= u && u <= 0xDFFFul) + return replacementCharacter; + // 0xFFFE: byte order marker (invalid character) + // 0xFFFF: permanently unassigned code point, may correspond to WEOF + // (raises errors when used in VC's swprintf) + // greater: UTF-16 representation would require surrogates + if(u >= 0xFFFEul) + return replacementCharacter; + return u; } @@ -84,21 +79,26 @@ public: { const size_t size = Size(u); static const UTF8 firstByteMarks[1+3] = { 0, 0x00, 0xC0, 0xE0 }; - *dstPos++ = (UTF8)(u | firstByteMarks[size]); for(size_t i = 1; i < size; i++) { - *dstPos++ = (UTF8)((u|0x80u) & 0xBFu); + dstPos[size-i] = UTF8((u|0x80u) & 0xBFu); u >>= 6; } + dstPos[0] = UTF8(u | firstByteMarks[size]); + dstPos += size; } - static bool Decode(const UTF8*& srcPos, const UTF8* const srcEnd, UTF32& u) + // @return decoded scalar, or replacementCharacter on error + static UTF32 Decode(const UTF8*& srcPos, const UTF8* const srcEnd) { const size_t size = SizeFromFirstByte(*srcPos); if(!IsValid(srcPos, size, srcEnd)) - return false; + { + srcPos += 1; // only skip the offending byte (increases chances of resynchronization) + return replacementCharacter; + } - u = 0; + UTF32 u = 0; for(size_t i = 0; i < size-1; i++) { u += UTF32(*srcPos++); @@ -106,10 +106,9 @@ public: } u += UTF32(*srcPos++); - static const UTF32 offsets[1+3] = { 0, 0x00000000ul, 0x00003080ul, 0x000E2080ul }; + static const UTF32 offsets[1+4] = { 0, 0x00000000ul, 0x00003080ul, 0x000E2080ul, 0x03C82080UL }; u -= offsets[size]; - - return true; + return u; } private: @@ -129,8 +128,10 @@ private: return 1; if(firstByte < 0xE0) return 2; - // IsValid rejects firstByte values that would cause > 3 byte encodings. - return 3; + if(firstByte < 0xF0) + return 3; + // IsValid rejects firstByte values that would cause > 4 byte encodings. + return 4; } // c.f. Unicode 3.1 Table 3-7 @@ -142,7 +143,7 @@ private: if(src[0] < 0x80) return true; - if(!(0xC2 <= src[0] && src[0] <= 0xEF)) + if(!(0xC2 <= src[0] && src[0] <= 0xF4)) return false; // special cases (stricter than the loop) @@ -150,6 +151,10 @@ private: return false; if(src[0] == 0xED && src[1] > 0x9F) return false; + if(src[0] == 0xF0 && src[1] < 0x90) + return false; + if(src[0] == 0xF4 && src[1] > 0x8F) + return false; for(size_t i = 1; i < size; i++) { @@ -164,9 +169,9 @@ private: //----------------------------------------------------------------------------- -std::string UTF8_from_wstring(const std::wstring& src) +std::string utf8_from_wstring(const std::wstring& src) { - std::string dst(src.size()*3, ' '); // see UTF8Codec::Size + std::string dst(src.size()*3+1, ' '); // see UTF8Codec::Size; +1 ensures &dst[0] is valid UTF8* dstPos = (UTF8*)&dst[0]; for(size_t i = 0; i < src.size(); i++) { @@ -178,7 +183,7 @@ std::string UTF8_from_wstring(const std::wstring& src) } -std::wstring wstring_from_UTF8(const std::string& src) +std::wstring wstring_from_utf8(const std::string& src) { std::wstring dst; dst.reserve(src.size()); @@ -186,12 +191,7 @@ std::wstring wstring_from_UTF8(const std::string& src) const UTF8* const srcEnd = srcPos + src.size(); while(srcPos < srcEnd) { - UTF32 u; - if(!UTF8Codec::Decode(srcPos, srcEnd, u)) - { - debug_assert(0); - return L"(wstring_from_UTF8: invalid input)"; - } + const UTF32 u = UTF8Codec::Decode(srcPos, srcEnd); dst.push_back((wchar_t)ReplaceIfInvalid(u)); } return dst; diff --git a/source/lib/wchar.h b/source/lib/wchar.h index f04af3c962..2c663099e4 100644 --- a/source/lib/wchar.h +++ b/source/lib/wchar.h @@ -18,7 +18,7 @@ #ifndef INCLUDED_WCHAR #define INCLUDED_WCHAR -LIB_API std::wstring wstring_from_UTF8(const std::string& s); -LIB_API std::string UTF8_from_wstring(const std::wstring& s); +LIB_API std::wstring wstring_from_utf8(const std::string& s); +LIB_API std::string utf8_from_wstring(const std::wstring& s); #endif // #ifndef INCLUDED_WCHAR diff --git a/source/network/NetLog.cpp b/source/network/NetLog.cpp index 541f35951a..0a20b30fcf 100644 --- a/source/network/NetLog.cpp +++ b/source/network/NetLog.cpp @@ -375,7 +375,7 @@ void CNetLogFileSink::OpenFile( const fs::wpath& fileName, bool append ) if ( m_File.is_open() ) m_File.close(); // Open the file and log start - m_File.open( UTF8_from_wstring(fileName.string()).c_str(), append ? std::ios::app : std::ios::out ); + m_File.open( utf8_from_wstring(fileName.string()).c_str(), append ? std::ios::app : std::ios::out ); if ( !m_File.is_open() ) { // throw std::ios_base::failure diff --git a/source/ps/CLogger.cpp b/source/ps/CLogger.cpp index 38da0c51c6..c578e44836 100644 --- a/source/ps/CLogger.cpp +++ b/source/ps/CLogger.cpp @@ -56,10 +56,10 @@ const wchar_t* html_footer = L""; CLogger::CLogger() { fs::wpath mainlogPath(psLogDir()/L"mainlog.html"); - m_MainLog = new std::wofstream(UTF8_from_wstring(mainlogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); + m_MainLog = new std::wofstream(utf8_from_wstring(mainlogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); fs::wpath interestinglogPath(psLogDir()/L"interestinglog.html"); - m_InterestingLog = new std::wofstream(UTF8_from_wstring(interestinglogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); + m_InterestingLog = new std::wofstream(utf8_from_wstring(interestinglogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); m_OwnsStreams = true; m_UseDebugPrintf = true; diff --git a/source/ps/GameSetup/Config.cpp b/source/ps/GameSetup/Config.cpp index c2b1414765..5a0697b5e9 100644 --- a/source/ps/GameSetup/Config.cpp +++ b/source/ps/GameSetup/Config.cpp @@ -74,7 +74,7 @@ CStr g_AutostartMap = ""; static void LoadProfile( const CStr& profile ) { - VfsPath path = VfsPath(L"profiles") / wstring_from_UTF8(profile); + VfsPath path = VfsPath(L"profiles") / wstring_from_utf8(profile); VfsPath configFilename = path / L"settings/user.cfg"; g_ConfigDB.SetConfigFile(CFG_USER, true, configFilename.string().c_str()); diff --git a/source/ps/GameSetup/GameSetup.cpp b/source/ps/GameSetup/GameSetup.cpp index bc955c2f36..3afc191d0d 100644 --- a/source/ps/GameSetup/GameSetup.cpp +++ b/source/ps/GameSetup/GameSetup.cpp @@ -567,7 +567,7 @@ static void InitVfs(const CmdLineArgs& args) { size_t priority = i; int flags = VFS_MOUNT_WATCH|VFS_MOUNT_ARCHIVABLE; - std::wstring modName (wstring_from_UTF8(mods[i])); + std::wstring modName (wstring_from_utf8(mods[i])); g_VFS->Mount(L"", AddSlash(modLoosePath/modName), flags, priority); g_VFS->Mount(L"", AddSlash(modArchivePath/modName), flags, priority); } diff --git a/source/ps/GameSetup/Paths.cpp b/source/ps/GameSetup/Paths.cpp index 0c3065859b..2983a8023b 100644 --- a/source/ps/GameSetup/Paths.cpp +++ b/source/ps/GameSetup/Paths.cpp @@ -51,7 +51,7 @@ Paths::Paths(const CmdLineArgs& args) #else const char* envHome = getenv("HOME"); debug_assert(envHome); - const fs::wpath home(wstring_from_UTF8(envHome)); + const fs::wpath home(wstring_from_utf8(envHome)); m_data = AddSlash(XDG_Path("XDG_DATA_HOME", home, home/L".local/share/")/subdirectoryName); m_config = AddSlash(XDG_Path("XDG_CONFIG_HOME", home, home/L".config/")/subdirectoryName); m_cache = AddSlash(XDG_Path("XDG_CACHE_HOME", home, home/L".cache/")/subdirectoryName); @@ -92,8 +92,8 @@ Paths::Paths(const CmdLineArgs& args) if(path) { if(path[0] != '/') // relative to $HOME - return AddSlash(home/wstring_from_UTF8(path)); - return AddSlash(fs::wpath(wstring_from_UTF8(path))); + return AddSlash(home/wstring_from_utf8(path)); + return AddSlash(fs::wpath(wstring_from_utf8(path))); } return AddSlash(defaultPath); } diff --git a/source/ps/ProfileViewer.cpp b/source/ps/ProfileViewer.cpp index b9db753bef..2f303c5931 100644 --- a/source/ps/ProfileViewer.cpp +++ b/source/ps/ProfileViewer.cpp @@ -432,7 +432,7 @@ void CProfileViewer::SaveToFile() // Open the file. (It will be closed when the CProfileViewer // destructor is called.) fs::wpath path(psLogDir()/L"profile.txt"); - m->outputStream.open(UTF8_from_wstring(path.string()).c_str(), std::ofstream::out | std::ofstream::trunc); + m->outputStream.open(utf8_from_wstring(path.string()).c_str(), std::ofstream::out | std::ofstream::trunc); if (m->outputStream.fail()) { diff --git a/source/ps/i18n.cpp b/source/ps/i18n.cpp index 050f617dcb..f9b8380a60 100644 --- a/source/ps/i18n.cpp +++ b/source/ps/i18n.cpp @@ -59,7 +59,7 @@ bool I18n::LoadLanguage(const char* name) // Automatically delete the pointer when returning early std::auto_ptr locale (locale_ptr); - VfsPath dirname = AddSlash(VfsPath(L"language")/wstring_from_UTF8(name)); + VfsPath dirname = AddSlash(VfsPath(L"language")/wstring_from_utf8(name)); // Open *.lng with LoadStrings VfsPaths pathnames;