diff --git a/source/gui/CGUI.cpp b/source/gui/CGUI.cpp index b4778977e3..076818fea0 100644 --- a/source/gui/CGUI.cpp +++ b/source/gui/CGUI.cpp @@ -1557,7 +1557,7 @@ void CGUI::Xeromyces_ReadImage(XMBElement Element, CXeromyces* pFile, CGUISprite if (attr_name == "texture") { - image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_string(attr_value); + image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_UTF8(attr_value); } else if (attr_name == "size") diff --git a/source/gui/GUIRenderer.cpp b/source/gui/GUIRenderer.cpp index ec2abd94da..2994d5e32e 100644 --- a/source/gui/GUIRenderer.cpp +++ b/source/gui/GUIRenderer.cpp @@ -370,7 +370,7 @@ void GUIRenderer::UpdateDrawCallCache(DrawCalls &Calls, const CStr& SpriteName, if (SpriteName.substr(0, 10) == "stretched:") { SGUIImage Image; - Image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_string(SpriteName.substr(10)); + Image.m_TextureName = VfsPath(L"art/textures/ui")/wstring_from_UTF8(SpriteName.substr(10)); CClientArea ca("0 0 100% 100%"); Image.m_Size = ca; Image.m_TextureSize = ca; diff --git a/source/lib/file/archive/archive_zip.cpp b/source/lib/file/archive/archive_zip.cpp index b0ab02c657..933252aa2c 100644 --- a/source/lib/file/archive/archive_zip.cpp +++ b/source/lib/file/archive/archive_zip.cpp @@ -27,7 +27,7 @@ #include "lib/bits.h" #include "lib/byte_order.h" -#include "lib/wchar.h" // wstring_from_string +#include "lib/wchar.h" // wstring_from_UTF8 #include "lib/fat_time.h" #include "lib/path_util.h" #include "lib/allocators/pool.h" @@ -135,7 +135,7 @@ public: { const size_t length = (size_t)read_le16(&m_fn_len); const char* pathname = (const char*)this + sizeof(CDFH); // not 0-terminated! - return wstring_from_string(std::string(pathname, length)); + return wstring_from_UTF8(std::string(pathname, length)); } off_t HeaderOffset() const diff --git a/source/lib/file/file_system.cpp b/source/lib/file/file_system.cpp index 1605e3a157..67dd80336b 100644 --- a/source/lib/file/file_system.cpp +++ b/source/lib/file/file_system.cpp @@ -23,7 +23,7 @@ #include #include "lib/path_util.h" -#include "lib/wchar.h" // wstring_from_string +#include "lib/wchar.h" // wstring_from_UTF8 #include "lib/posix/posix_filesystem.h" @@ -67,7 +67,7 @@ LibError GetDirectoryEntries(const fs::wpath& path, FileInfos* files, DirectoryN return LibError_from_errno(); } - const std::wstring name = wstring_from_string(osEnt->d_name); + const std::wstring name = wstring_from_UTF8(osEnt->d_name); RETURN_ERR(path_component_validate(name.c_str())); // get file information (mode, size, mtime) diff --git a/source/lib/path_util.cpp b/source/lib/path_util.cpp index a90ff11c49..4e00121616 100644 --- a/source/lib/path_util.cpp +++ b/source/lib/path_util.cpp @@ -128,10 +128,10 @@ const wchar_t* path_name_only(const wchar_t* path) fs::wpath wpath_from_path(const fs::path& pathname) { - return wstring_from_string(pathname.string()); + return wstring_from_UTF8(pathname.string()); } fs::path path_from_wpath(const fs::wpath& pathname) { - return string_from_wstring(pathname.string()); + return UTF8_from_wstring(pathname.string()); } diff --git a/source/lib/secure_crt.cpp b/source/lib/secure_crt.cpp index 9ab811fac0..27d8447c6b 100644 --- a/source/lib/secure_crt.cpp +++ b/source/lib/secure_crt.cpp @@ -257,8 +257,8 @@ int tsprintf_s(tchar* buf, size_t max_chars, const tchar* fmt, ...) errno_t _wfopen_s(FILE** pfile, const wchar_t* filename, const wchar_t* mode) { *pfile = NULL; - const std::string filename_c = string_from_wstring(filename); - const std::string mode_c = string_from_wstring(mode); + const std::string filename_c = UTF8_from_wstring(filename); + const std::string mode_c = UTF8_from_wstring(mode); return fopen_s(pfile, filename_c.c_str(), mode_c.c_str()); } diff --git a/source/lib/sysdep/os/win/wseh.cpp b/source/lib/sysdep/os/win/wseh.cpp index 0d724cc06a..31c5999cac 100644 --- a/source/lib/sysdep/os/win/wseh.cpp +++ b/source/lib/sysdep/os/win/wseh.cpp @@ -281,7 +281,7 @@ long __stdcall wseh_ExceptionFilter(struct _EXCEPTION_POINTERS* ep) if(ep->ExceptionRecord->ExceptionFlags & EXCEPTION_NONCONTINUABLE) flags = DE_NO_CONTINUE; const wchar_t* const lastFuncToSkip = WIDEN(STRINGIZE(DECORATED_NAME(wseh_ExceptionFilter))); - ErrorReaction er = debug_DisplayError(message, flags, ep->ContextRecord, lastFuncToSkip, file,line,string_from_wstring(func).c_str(), 0); + ErrorReaction er = debug_DisplayError(message, flags, ep->ContextRecord, lastFuncToSkip, file,line,UTF8_from_wstring(func).c_str(), 0); debug_assert(er == ER_CONTINUE); // nothing else possible // invoke the Win32 default handler - it calls ExitProcess for diff --git a/source/lib/sysdep/tests/test_sysdep.h b/source/lib/sysdep/tests/test_sysdep.h index 3f23fd815a..a4d3518362 100644 --- a/source/lib/sysdep/tests/test_sysdep.h +++ b/source/lib/sysdep/tests/test_sysdep.h @@ -108,7 +108,7 @@ public: sprintf_s(root, ARRAY_SIZE(root), "%s/pyrogenesis-test-sysdep-XXXXXX", tmpdir); TS_ASSERT(mkdtemp(root)); std::string rootstr(root); - std::wstring rootstrw(wstring_from_string(rootstr)); + std::wstring rootstrw(wstring_from_UTF8(rootstr)); const char* dirs[] = { "/example", diff --git a/source/lib/wchar.cpp b/source/lib/wchar.cpp index 69b763dc67..a9ec036331 100644 --- a/source/lib/wchar.cpp +++ b/source/lib/wchar.cpp @@ -1,37 +1,198 @@ /* Copyright (C) 2009 Wildfire Games. - * This file is part of 0 A.D. - * - * 0 A.D. is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * - * 0 A.D. is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with 0 A.D. If not, see . - */ +* This file is part of 0 A.D. +* +* 0 A.D. is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 2 of the License, or +* (at your option) any later version. +* +* 0 A.D. is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with 0 A.D. If not, see . +*/ #include "precompiled.h" #include "wchar.h" +// adapted from http://unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c +// which bears the following notice: +/* +* Copyright 2001-2004 Unicode, Inc. +* +* Disclaimer +* +* This source code is provided as is by Unicode, Inc. No claims are +* made as to fitness for any particular purpose. No warranties of any +* kind are expressed or implied. The recipient agrees to determine +* applicability of information provided. If this file has been +* purchased on magnetic or optical media from Unicode, Inc., the +* sole remedy for any claim will be exchange of defective media +* within 90 days of receipt. +* +* Limitations on Rights to Redistribute This Code +* +* Unicode, Inc. hereby grants the right to freely use the information +* supplied in this file in the creation of products supporting the +* Unicode Standard, and to make copies of this file in any form +* for internal or external distribution as long as this notice +* remains attached. +*/ -std::wstring wstring_from_string(const std::string& s) +// design rationale: +// - to cope with wchar_t differences between VC (UTF-16) and +// GCC (UCS-4), we only allow codepoints in the BMP. +// UTF-8 encodings are therefore no longer than 3 bytes. +// - surrogates are disabled because variable-length strings +// violate the purpose of using wchar_t instead of UTF-8. + +// this implementation survives http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + +// (must be unsigned to avoid sign extension) +typedef u8 UTF8; +typedef u32 UTF32; + + +static UTF32 ReplaceIfInvalid(UTF32 u) { - wchar_t buf[1000]; - const size_t numConverted = mbstowcs(buf, s.c_str(), ARRAY_SIZE(buf)-1); - debug_assert(numConverted < ARRAY_SIZE(buf)); - return buf; + struct IsValid + { + bool operator()(UTF32 u) const + { + // disallow surrogates + if(0xDC00ul <= u && u <= 0xDFFFul) + return false; + // greater: UTF-16 representation would require surrogates + // equal: permanently unassigned codepoint, may correspond to WEOF + // (which raises errors when used in VC's swprintf) + if(u >= 0xFFFFul) + return false; + return true; + } + }; + return IsValid()(u)? u : 0xFFFD; } -std::string string_from_wstring(const std::wstring& s) +class UTF8Codec { - char buf[1000]; - const size_t numConverted = wcstombs(buf, s.c_str(), ARRAY_SIZE(buf)-1); - debug_assert(numConverted < ARRAY_SIZE(buf)); - return buf; +public: + static void Encode(UTF32 u, UTF8*& dstPos) + { + const size_t size = Size(u); + static const UTF8 firstByteMarks[1+3] = { 0, 0x00, 0xC0, 0xE0 }; + *dstPos++ = (UTF8)(u | firstByteMarks[size]); + for(size_t i = 1; i < size; i++) + { + *dstPos++ = (UTF8)((u|0x80u) & 0xBFu); + u >>= 6; + } + } + + static bool Decode(const UTF8*& srcPos, const UTF8* const srcEnd, UTF32& u) + { + const size_t size = SizeFromFirstByte(*srcPos); + if(!IsValid(srcPos, size, srcEnd)) + return false; + + u = 0; + for(size_t i = 0; i < size-1; i++) + { + u += UTF32(*srcPos++); + u <<= 6; + } + u += UTF32(*srcPos++); + + static const UTF32 offsets[1+3] = { 0, 0x00000000ul, 0x00003080ul, 0x000E2080ul }; + u -= offsets[size]; + + return true; + } + +private: + static inline size_t Size(UTF32 u) + { + if(u < 0x80) + return 1; + if(u < 0x800) + return 2; + // ReplaceIfInvalid ensures > 3 byte encodings are never used. + return 3; + } + + static inline size_t SizeFromFirstByte(UTF8 firstByte) + { + if(firstByte < 0xC0) + return 1; + if(firstByte < 0xE0) + return 2; + // IsValid rejects firstByte values that would cause > 3 byte encodings. + return 3; + } + + // c.f. Unicode 3.1 Table 3-7 + // @param size obtained via SizeFromFirstByte (our caller also uses it) + static bool IsValid(const UTF8* const src, size_t size, const UTF8* const srcEnd) + { + if(src+size > srcEnd) // not enough data + return false; + + if(src[0] < 0x80) + return true; + if(!(0xC2 <= src[0] && src[0] <= 0xEF)) + return false; + + // special cases (stricter than the loop) + if(src[0] == 0xE0 && src[1] < 0xA0) + return false; + if(src[0] == 0xED && src[1] > 0x9F) + return false; + + for(size_t i = 1; i < size; i++) + { + if(!(0x80 <= src[i] && src[i] <= 0xBF)) + return false; + } + + return true; + } +}; + + +//----------------------------------------------------------------------------- + +std::string UTF8_from_wstring(const std::wstring& src) +{ + std::string dst(src.size()*3, ' '); // see UTF8Codec::Size + UTF8* dstPos = (UTF8*)&dst[0]; + for(size_t i = 0; i < src.size(); i++) + { + const UTF32 u = ReplaceIfInvalid(UTF32(src[i])); + UTF8Codec::Encode(u, dstPos); + } + dst.resize(dstPos - (UTF8*)&dst[0]); + return dst; +} + + +std::wstring wstring_from_UTF8(const std::string& src) +{ + std::wstring dst; + dst.reserve(src.size()); + const UTF8* srcPos = (const UTF8*)src.data(); + const UTF8* const srcEnd = srcPos + src.size(); + while(srcPos < srcEnd) + { + UTF32 u; + if(!UTF8Codec::Decode(srcPos, srcEnd, u)) + { + debug_assert(0); + return L"(wstring_from_UTF8: invalid input)"; + } + dst.push_back((wchar_t)ReplaceIfInvalid(u)); + } + return dst; } diff --git a/source/lib/wchar.h b/source/lib/wchar.h index e3dd5ff8b4..f04af3c962 100644 --- a/source/lib/wchar.h +++ b/source/lib/wchar.h @@ -18,7 +18,7 @@ #ifndef INCLUDED_WCHAR #define INCLUDED_WCHAR -LIB_API std::wstring wstring_from_string(const std::string& s); -LIB_API std::string string_from_wstring(const std::wstring& s); +LIB_API std::wstring wstring_from_UTF8(const std::string& s); +LIB_API std::string UTF8_from_wstring(const std::wstring& s); #endif // #ifndef INCLUDED_WCHAR diff --git a/source/network/NetLog.cpp b/source/network/NetLog.cpp index dbd93e17ef..541f35951a 100644 --- a/source/network/NetLog.cpp +++ b/source/network/NetLog.cpp @@ -375,7 +375,7 @@ void CNetLogFileSink::OpenFile( const fs::wpath& fileName, bool append ) if ( m_File.is_open() ) m_File.close(); // Open the file and log start - m_File.open( string_from_wstring(fileName.string()).c_str(), append ? std::ios::app : std::ios::out ); + m_File.open( UTF8_from_wstring(fileName.string()).c_str(), append ? std::ios::app : std::ios::out ); if ( !m_File.is_open() ) { // throw std::ios_base::failure diff --git a/source/ps/CLogger.cpp b/source/ps/CLogger.cpp index 10fced6f67..38da0c51c6 100644 --- a/source/ps/CLogger.cpp +++ b/source/ps/CLogger.cpp @@ -56,10 +56,10 @@ const wchar_t* html_footer = L""; CLogger::CLogger() { fs::wpath mainlogPath(psLogDir()/L"mainlog.html"); - m_MainLog = new std::wofstream(string_from_wstring(mainlogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); + m_MainLog = new std::wofstream(UTF8_from_wstring(mainlogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); fs::wpath interestinglogPath(psLogDir()/L"interestinglog.html"); - m_InterestingLog = new std::wofstream(string_from_wstring(interestinglogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); + m_InterestingLog = new std::wofstream(UTF8_from_wstring(interestinglogPath.string()).c_str(), std::ofstream::out | std::ofstream::trunc); m_OwnsStreams = true; m_UseDebugPrintf = true; diff --git a/source/ps/GameSetup/Config.cpp b/source/ps/GameSetup/Config.cpp index ded66be390..c2b1414765 100644 --- a/source/ps/GameSetup/Config.cpp +++ b/source/ps/GameSetup/Config.cpp @@ -74,7 +74,7 @@ CStr g_AutostartMap = ""; static void LoadProfile( const CStr& profile ) { - VfsPath path = VfsPath(L"profiles") / wstring_from_string(profile); + VfsPath path = VfsPath(L"profiles") / wstring_from_UTF8(profile); VfsPath configFilename = path / L"settings/user.cfg"; g_ConfigDB.SetConfigFile(CFG_USER, true, configFilename.string().c_str()); diff --git a/source/ps/GameSetup/GameSetup.cpp b/source/ps/GameSetup/GameSetup.cpp index 57499b2ac3..bc955c2f36 100644 --- a/source/ps/GameSetup/GameSetup.cpp +++ b/source/ps/GameSetup/GameSetup.cpp @@ -567,7 +567,7 @@ static void InitVfs(const CmdLineArgs& args) { size_t priority = i; int flags = VFS_MOUNT_WATCH|VFS_MOUNT_ARCHIVABLE; - std::wstring modName (wstring_from_string(mods[i])); + std::wstring modName (wstring_from_UTF8(mods[i])); g_VFS->Mount(L"", AddSlash(modLoosePath/modName), flags, priority); g_VFS->Mount(L"", AddSlash(modArchivePath/modName), flags, priority); } diff --git a/source/ps/GameSetup/Paths.cpp b/source/ps/GameSetup/Paths.cpp index f087ca3ce8..0c3065859b 100644 --- a/source/ps/GameSetup/Paths.cpp +++ b/source/ps/GameSetup/Paths.cpp @@ -51,7 +51,7 @@ Paths::Paths(const CmdLineArgs& args) #else const char* envHome = getenv("HOME"); debug_assert(envHome); - const fs::wpath home(wstring_from_string(envHome)); + const fs::wpath home(wstring_from_UTF8(envHome)); m_data = AddSlash(XDG_Path("XDG_DATA_HOME", home, home/L".local/share/")/subdirectoryName); m_config = AddSlash(XDG_Path("XDG_CONFIG_HOME", home, home/L".config/")/subdirectoryName); m_cache = AddSlash(XDG_Path("XDG_CACHE_HOME", home, home/L".cache/")/subdirectoryName); @@ -92,8 +92,8 @@ Paths::Paths(const CmdLineArgs& args) if(path) { if(path[0] != '/') // relative to $HOME - return AddSlash(home/wstring_from_string(path)); - return AddSlash(fs::wpath(wstring_from_string(path))); + return AddSlash(home/wstring_from_UTF8(path)); + return AddSlash(fs::wpath(wstring_from_UTF8(path))); } return AddSlash(defaultPath); } diff --git a/source/ps/ProfileViewer.cpp b/source/ps/ProfileViewer.cpp index acd2b1e7ca..b9db753bef 100644 --- a/source/ps/ProfileViewer.cpp +++ b/source/ps/ProfileViewer.cpp @@ -432,7 +432,7 @@ void CProfileViewer::SaveToFile() // Open the file. (It will be closed when the CProfileViewer // destructor is called.) fs::wpath path(psLogDir()/L"profile.txt"); - m->outputStream.open(string_from_wstring(path.string()).c_str(), std::ofstream::out | std::ofstream::trunc); + m->outputStream.open(UTF8_from_wstring(path.string()).c_str(), std::ofstream::out | std::ofstream::trunc); if (m->outputStream.fail()) { diff --git a/source/ps/i18n.cpp b/source/ps/i18n.cpp index 3d543f39af..050f617dcb 100644 --- a/source/ps/i18n.cpp +++ b/source/ps/i18n.cpp @@ -59,7 +59,7 @@ bool I18n::LoadLanguage(const char* name) // Automatically delete the pointer when returning early std::auto_ptr locale (locale_ptr); - VfsPath dirname = AddSlash(VfsPath(L"language")/wstring_from_string(name)); + VfsPath dirname = AddSlash(VfsPath(L"language")/wstring_from_UTF8(name)); // Open *.lng with LoadStrings VfsPaths pathnames;