diff --git a/source/graphics/MapGenerator.cpp b/source/graphics/MapGenerator.cpp index f03d1dbcd4..518e98e13f 100644 --- a/source/graphics/MapGenerator.cpp +++ b/source/graphics/MapGenerator.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2011 Wildfire Games. +/* Copyright (C) 2012 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -199,7 +199,7 @@ std::vector CMapGeneratorWorker::GetCivData(void* UNUSED(cbdata)) } else { - data.push_back(std::string(file.GetBuffer(), file.GetBuffer() + file.GetBufferSize())); + data.push_back(file.DecodeUTF8()); // assume it's UTF-8 } } } diff --git a/source/ps/Filesystem.cpp b/source/ps/Filesystem.cpp index acd83441e1..9f31af9417 100644 --- a/source/ps/Filesystem.cpp +++ b/source/ps/Filesystem.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2010 Wildfire Games. +/* Copyright (C) 2012 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -24,6 +24,7 @@ #include "lib/res/h_mgr.h" // h_reload #include "lib/sysdep/dir_watch.h" +#include "lib/utf8.h" PIVFS g_VFS; @@ -139,3 +140,18 @@ CStr CVFSFile::GetAsString() const { return std::string((char*)GetBuffer(), GetBufferSize()); } + +CStr CVFSFile::DecodeUTF8() const +{ + const u8* buffer = GetBuffer(); + + // Detect if there's a UTF-8 BOM and strip it + if (GetBufferSize() >= 3 && buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) + { + return std::string(&buffer[3], buffer + GetBufferSize()); + } + else + { + return std::string(buffer, buffer + GetBufferSize()); + } +} diff --git a/source/ps/Filesystem.h b/source/ps/Filesystem.h index fbf7180c8a..340c5bac60 100644 --- a/source/ps/Filesystem.h +++ b/source/ps/Filesystem.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2010 Wildfire Games. +/* Copyright (C) 2012 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -58,22 +58,40 @@ ERROR_GROUP(CVFSFile); ERROR_TYPE(CVFSFile, LoadFailed); ERROR_TYPE(CVFSFile, AlreadyLoaded); -// Reads a file, then gives read-only access to the contents +/** + * Reads a file, then gives read-only access to the contents + */ class CVFSFile { public: CVFSFile(); ~CVFSFile(); - // Returns either PSRETURN_OK or PSRETURN_CVFSFile_LoadFailed. - // Dies if a file has already been successfully loaded. + /** + * Returns either PSRETURN_OK or PSRETURN_CVFSFile_LoadFailed + * @note Dies if the file has already been successfully loaded + */ PSRETURN Load(const PIVFS& vfs, const VfsPath& filename); - // These die if called when no file has been successfully loaded. + /** + * Returns buffer of this file as a stream of bytes + * @note file must have been successfully loaded + */ const u8* GetBuffer() const; size_t GetBufferSize() const; + + /** + * Returns contents of file as a string + * @note file must have been successfully loaded + */ CStr GetAsString() const; + /** + * Returns contents of a UTF-8 encoded file as a string with optional BOM removed + * @note file must have been successfully loaded + */ + CStr DecodeUTF8() const; + private: shared_ptr m_Buffer; size_t m_BufferSize; diff --git a/source/ps/scripting/JSInterface_VFS.cpp b/source/ps/scripting/JSInterface_VFS.cpp index 24d8f3ede7..3ba402b045 100644 --- a/source/ps/scripting/JSInterface_VFS.cpp +++ b/source/ps/scripting/JSInterface_VFS.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2009 Wildfire Games. +/* Copyright (C) 2012 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -170,7 +170,7 @@ JSBool JSI_VFS::GetFileSize(JSContext* cx, uintN argc, jsval* vp) } -// Return file contents in a string. +// Return file contents in a string. Assume file is UTF-8 encoded text. // // contents = readFile(filename); // filename: VFS filename (may include path) @@ -182,12 +182,17 @@ JSBool JSI_VFS::ReadFile(JSContext* cx, uintN argc, jsval* vp) if (!ToPrimitive (cx, JS_ARGV(cx, vp)[0], filename)) return JS_FALSE; - shared_ptr buf; - size_t size; - Status err = g_VFS->LoadFile(filename, buf, size); - JS_CHECK_FILE_ERR( err ); + // + // read file + // + CVFSFile file; + if (file.Load(g_VFS, filename) != PSRETURN_OK) + { + JS_SET_RVAL(cx, vp, JSVAL_NULL); + return JS_TRUE; + } - CStr contents((const char*)buf.get(), size); + CStr contents = file.DecodeUTF8(); // assume it's UTF-8 // Fix CRLF line endings. (This function will only ever be used on text files.) contents.Replace("\r\n", "\n"); @@ -198,7 +203,7 @@ JSBool JSI_VFS::ReadFile(JSContext* cx, uintN argc, jsval* vp) } -// Return file contents as an array of lines. +// Return file contents as an array of lines. Assume file is UTF-8 encoded text. // // lines = readFileLines(filename); // filename: VFS filename (may include path) @@ -213,13 +218,14 @@ JSBool JSI_VFS::ReadFileLines(JSContext* cx, uintN argc, jsval* vp) // // read file // + CVFSFile file; + if (file.Load(g_VFS, filename) != PSRETURN_OK) + { + JS_SET_RVAL(cx, vp, JSVAL_NULL); + return JS_TRUE; + } - shared_ptr buf; - size_t size; - Status err = g_VFS->LoadFile(filename, buf, size); - JS_CHECK_FILE_ERR( err ); - - CStr contents((const char*)buf.get(), size); + CStr contents = file.DecodeUTF8(); // assume it's UTF-8 // Fix CRLF line endings. (This function will only ever be used on text files.) contents.Replace("\r\n", "\n"); diff --git a/source/scriptinterface/ScriptInterface.cpp b/source/scriptinterface/ScriptInterface.cpp index 4ad6f6748f..e33bb9a570 100644 --- a/source/scriptinterface/ScriptInterface.cpp +++ b/source/scriptinterface/ScriptInterface.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2011 Wildfire Games. +/* Copyright (C) 2012 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -1018,7 +1018,7 @@ CScriptValRooted ScriptInterface::ReadJSONFile(const VfsPath& path) return CScriptValRooted(); } - std::string content(file.GetBuffer(), file.GetBuffer() + file.GetBufferSize()); // assume it's UTF-8 + std::string content(file.DecodeUTF8()); // assume it's UTF-8 return ParseJSON(content); } diff --git a/source/simulation2/Simulation2.cpp b/source/simulation2/Simulation2.cpp index 62840e40ee..45b1362324 100644 --- a/source/simulation2/Simulation2.cpp +++ b/source/simulation2/Simulation2.cpp @@ -798,7 +798,7 @@ std::vector CSimulation2::GetRMSData() } else { - data.push_back(std::string(file.GetBuffer(), file.GetBuffer() + file.GetBufferSize())); + data.push_back(file.DecodeUTF8()); // assume it's UTF-8 } } } @@ -834,7 +834,7 @@ std::vector CSimulation2::GetCivData() } else { - data.push_back(std::string(file.GetBuffer(), file.GetBuffer() + file.GetBufferSize())); + data.push_back(file.DecodeUTF8()); // assume it's UTF-8 } } } @@ -877,7 +877,7 @@ std::string CSimulation2::ReadJSON(VfsPath path) } else { - data = std::string(file.GetBuffer(), file.GetBuffer() + file.GetBufferSize()); + data = file.DecodeUTF8(); // assume it's UTF-8 } }