Convert CConsole to take UTF-8 strings.

This avoids vswprintf failures when printing non-ASCII char* strings
from CLogger into the console.

Also convert ScriptInterface::ToString to return UTF-8, to avoid some
utf8_from_wstring calls.

Also remove some unused and redundant CConsole functions.

This was SVN commit r16333.

source/gui/scripting/ScriptFunctions.cpp

@@ -628,7 +628,7 @@ void ForceGC(ScriptInterface::CxPrivate* pCxPrivate)
 	double time = timer_Time();
 	JS_GC(pCxPrivate->pScriptInterface->GetJSRuntime());
 	time = timer_Time() - time;
-	g_Console->InsertMessage(L"Garbage collection completed in: %f", time);
+	g_Console->InsertMessage(fmt::sprintf("Garbage collection completed in: %f", time));
 }
 
 void DumpSimState(ScriptInterface::CxPrivate* UNUSED(pCxPrivate))

source/network/NetClient.cpp

@@ -196,19 +196,19 @@ void CNetClient::PushGuiMessage(const JS::HandleValue message)
 	m_GuiMessageQueue.push_back(JS::Heap<JS::Value>(message));
 }
 
-std::wstring CNetClient::TestReadGuiMessages()
+std::string CNetClient::TestReadGuiMessages()
 {
 	JSContext* cx = GetScriptInterface().GetContext();
 	JSAutoRequest rq(cx);
 	
-	std::wstring r;
+	std::string r;
 	JS::RootedValue msg(cx);
 	while (true)
 	{
 		GuiPoll(&msg);
 		if (msg.isUndefined())
 			break;
-		r += GetScriptInterface().ToString(&msg) + L"\n";
+		r += GetScriptInterface().ToString(&msg) + "\n";
 	}
 	return r;
 }

source/network/NetClient.h

@@ -140,7 +140,7 @@ public:
 	 * Return a concatenation of all messages in the GUI queue,
 	 * for test cases to easily verify the queue contents.
 	 */
-	std::wstring TestReadGuiMessages();
+	std::string TestReadGuiMessages();
 
 	/**
 	 * Get the script interface associated with this network client,

source/network/NetMessageSim.cpp

@@ -176,7 +176,7 @@ size_t CSimulationMessage::GetSerializedLength() const
 
 CStr CSimulationMessage::ToString() const
 {
-	std::string source = utf8_from_wstring(m_ScriptInterface->ToString(const_cast<JS::PersistentRootedValue*>(&m_Data)));
+	std::string source = m_ScriptInterface->ToString(const_cast<JS::PersistentRootedValue*>(&m_Data));
 
 	std::stringstream stream;
 	stream << "CSimulationMessage { m_Client: " << m_Client << ", m_Player: " << m_Player << ", m_Turn: " << m_Turn << ", m_Data: " << source << " }";
@@ -223,7 +223,7 @@ size_t CGameSetupMessage::GetSerializedLength() const
 
 CStr CGameSetupMessage::ToString() const
 {
-	std::string source = utf8_from_wstring(m_ScriptInterface.ToString(const_cast<JS::PersistentRootedValue*>(&m_Data)));
+	std::string source = m_ScriptInterface.ToString(const_cast<JS::PersistentRootedValue*>(&m_Data));
 
 	std::stringstream stream;
 	stream << "CGameSetupMessage { m_Data: " << source << " }";

source/network/tests/test_Net.h

@@ -161,7 +161,7 @@ public:
 		clients.push_back(&client3);
 
 		connect(server, clients);
-		debug_printf("%s", utf8_from_wstring(client1.TestReadGuiMessages()).c_str());
+		debug_printf("%s", client1.TestReadGuiMessages().c_str());
 
 		server.StartGame();
 		SDL_Delay(100);
@@ -230,7 +230,7 @@ public:
 		clients.push_back(&client3);
 
 		connect(server, clients);
-		debug_printf("%s", utf8_from_wstring(client1.TestReadGuiMessages()).c_str());
+		debug_printf("%s", client1.TestReadGuiMessages().c_str());
 
 		server.StartGame();
 		SDL_Delay(100);

source/ps/CConsole.cpp

@@ -63,8 +63,8 @@ CConsole::CConsole()
 	m_bCursorVisState = true;
 	m_cursorBlinkRate = 0.5;
 
-	InsertMessage(L"[ 0 A.D. Console v0.14 ]");
-	InsertMessage(L"");
+	InsertMessage("[ 0 A.D. Console v0.14 ]");
+	InsertMessage("");
 }
 
 CConsole::~CConsole()
@@ -127,47 +127,6 @@ void CConsole::FlushBuffer()
 }
 
 
-void CConsole::ToLower(wchar_t* szMessage, size_t iSize)
-{
-	size_t L = (size_t)wcslen(szMessage);
-
-	if (L <= 0) return;
-
-	if (iSize && iSize < L) L = iSize;
-
-	for(size_t i = 0; i < L; i++)
-		szMessage[i] = towlower(szMessage[i]);
-}
-
-
-void CConsole::Trim(wchar_t* szMessage, const wchar_t cChar, size_t iSize)
-{
-	size_t L = wcslen(szMessage);
-	if(!L)
-		return;
-
-	if (iSize && iSize < L) L = iSize;
-
-	wchar_t szChar[2] = { cChar, 0 };
-
-	// Find the first point at which szChar does not
-	// exist in the message
-	size_t ofs = wcsspn(szMessage, szChar);
-	if(ofs == 0)	// no leading <cChar> chars - we're done
-		return;
-
-	// move everything <ofs> chars left, replacing leading cChar chars
-	L -= ofs;
-	memmove(szMessage, szMessage+ofs, L*sizeof(wchar_t));
-
-	for(ssize_t i = (ssize_t)L; i >= 0; i--)
-	{
-		szMessage[i] = '\0';
-		if (szMessage[i - 1] != cChar) break;
-	}
-}
-
-
 void CConsole::Update(const float deltaRealTime)
 {
 	if(m_bToggle)
@@ -514,32 +473,13 @@ void CConsole::InsertChar(const int szChar, const wchar_t cooked)
 }
 
 
-void CConsole::InsertMessage(const wchar_t* szMessage, ...)
-{
-	va_list args;
-	wchar_t szBuffer[CONSOLE_MESSAGE_SIZE];
-
-	va_start(args, szMessage);
-	if (vswprintf(szBuffer, CONSOLE_MESSAGE_SIZE, szMessage, args) == -1)
-	{
-		debug_printf("Error printfing console message (buffer size exceeded?)\n");
-
-		// Make it obvious that the text was trimmed (assuming it was)
-		wcscpy(szBuffer+CONSOLE_MESSAGE_SIZE-4, L"...");
-	}
-	va_end(args);
-
-	InsertMessageRaw(szBuffer);
-}
-	
-
-void CConsole::InsertMessageRaw(const CStrW& message)
+void CConsole::InsertMessage(const std::string& message)
 {
 	// (TODO: this text-wrapping is rubbish since we now use variable-width fonts)
 
 	//Insert newlines to wraparound text where needed
-	CStrW wrapAround(message);
-	CStrW newline(L"\n");
+	std::wstring wrapAround = wstring_from_utf8(message.c_str());
+	std::wstring newline(L"\n");
 	size_t oldNewline=0;
 	size_t distance;
 	
@@ -620,7 +560,7 @@ void CConsole::ProcessBuffer(const wchar_t* szLine)
 	JS::RootedValue rval(cx);
 	pScriptInterface->Eval(szLine, &rval);
 	if (!rval.isUndefined())
-		InsertMessageRaw(pScriptInterface->ToString(&rval));
+		InsertMessage(pScriptInterface->ToString(&rval));
 }
 
 void CConsole::LoadHistory()
@@ -671,11 +611,6 @@ void CConsole::SaveHistory()
 	g_VFS->CreateFile(m_sHistoryFile, buffer.Data(), buffer.Size());
 }
 
-void CConsole::ReceivedChatMessage(const wchar_t *szSender, const wchar_t *szMessage)
-{
-	InsertMessage(L"%ls: %ls", szSender, szMessage);
-}
-
 #if SDL_VERSION_ATLEAST(2, 0, 0)
 static bool isUnprintableChar(SDL_Keysym key)
 #else

source/ps/CConsole.h

@@ -70,13 +70,9 @@ public:
 
 	void Render();
 
-	void InsertMessage(const wchar_t* szMessage, ...) WPRINTF_ARGS(2);
 	void InsertChar(const int szChar, const wchar_t cooked);
 
-	// Insert message without printf-style formatting, and without length limits
-	void InsertMessageRaw(const CStrW& message);
-
-	void ReceivedChatMessage(const wchar_t *pSender, const wchar_t *szMessage);
+	void InsertMessage(const std::string& message);
 
 	void SetBuffer(const wchar_t* szMessage);
 
@@ -126,9 +122,6 @@ private:
 	bool m_bCursorVisState;	// if the cursor should be drawn or not
 	double m_cursorBlinkRate;	// cursor blink rate in seconds, if greater than 0.0
 
-	void ToLower(wchar_t* szMessage, size_t iSize = 0);
-	void Trim(wchar_t* szMessage, const wchar_t cChar = 32, size_t iSize = 0);
-
 	void DrawWindow(CShaderProgramPtr& shader);
 	void DrawHistory(CTextRenderer& textRenderer);
 	void DrawBuffer(CTextRenderer& textRenderer);

source/ps/CLogger.cpp

@@ -155,7 +155,7 @@ void CLogger::WriteMessage(const char* message, bool doRender = false)
 	
 	if (doRender)
 	{
-		if (g_Console) g_Console->InsertMessage(L"INFO: %hs", message);
+		if (g_Console) g_Console->InsertMessage(std::string("INFO: ") + message);
 
 		PushRenderMessage(Normal, message);
 	}
@@ -171,7 +171,7 @@ void CLogger::WriteError(const char* message)
 	if (m_UseDebugPrintf)
 		debug_printf("ERROR: %s\n", message);
 
-	if (g_Console) g_Console->InsertMessage(L"ERROR: %hs", message);
+	if (g_Console) g_Console->InsertMessage(std::string("ERROR: ") + message);
 	*m_InterestingLog << "<p class=\"error\">ERROR: " << cmessage << "</p>\n";
 	m_InterestingLog->flush();
 
@@ -191,7 +191,7 @@ void CLogger::WriteWarning(const char* message)
 	if (m_UseDebugPrintf)
 		debug_printf("WARNING: %s\n", message);
 
-	if (g_Console) g_Console->InsertMessage(L"WARNING: %hs", message);
+	if (g_Console) g_Console->InsertMessage(std::string("WARNING: ") + message);
 	*m_InterestingLog << "<p class=\"warning\">WARNING: " << cmessage << "</p>\n";
 	m_InterestingLog->flush();
 

source/scriptinterface/ScriptInterface.cpp

@@ -1005,18 +1005,6 @@ struct Stringifier
 	std::stringstream stream;
 };
 
-struct StringifierW
-{
-	static bool callback(const jschar* buf, u32 len, void* data)
-	{
-		utf16string str(buf, buf+len);
-		static_cast<StringifierW*>(data)->stream << std::wstring(str.begin(), str.end());
-		return true;
-	}
-
-	std::wstringstream stream;
-};
-
 // TODO: It's not quite clear why JS_Stringify needs JS::MutableHandleValue. |obj| should not get modified.
 // It probably has historical reasons and could be changed by SpiderMonkey in the future.
 std::string ScriptInterface::StringifyJSON(JS::MutableHandleValue obj, bool indent)
@@ -1035,24 +1023,24 @@ std::string ScriptInterface::StringifyJSON(JS::MutableHandleValue obj, bool inde
 }
 
 
-std::wstring ScriptInterface::ToString(JS::MutableHandleValue obj, bool pretty)
+std::string ScriptInterface::ToString(JS::MutableHandleValue obj, bool pretty)
 {
 	JSAutoRequest rq(m->m_cx);
 
 	if (obj.isUndefined())
-		return L"(void 0)";
+		return "(void 0)";
 
 	// Try to stringify as JSON if possible
 	// (TODO: this is maybe a bad idea since it'll drop 'undefined' values silently)
 	if (pretty)
 	{
-		StringifierW str;
+		Stringifier str;
 		JS::RootedValue indentVal(m->m_cx, JS::Int32Value(2));
 
 		// Temporary disable the error reporter, so we don't print complaints about cyclic values
 		JSErrorReporter er = JS_SetErrorReporter(m->m_cx, NULL);
 
-		bool ok = JS_Stringify(m->m_cx, obj, JS::NullPtr(), indentVal, &StringifierW::callback, &str);
+		bool ok = JS_Stringify(m->m_cx, obj, JS::NullPtr(), indentVal, &Stringifier::callback, &str);
 
 		// Restore error reporter
 		JS_SetErrorReporter(m->m_cx, er);
@@ -1069,7 +1057,7 @@ std::wstring ScriptInterface::ToString(JS::MutableHandleValue obj, bool pretty)
 
 	std::wstring source = L"(error)";
 	CallFunction(obj, "toSource", source);
-	return source;
+	return utf8_from_wstring(source);
 }
 
 void ScriptInterface::ReportError(const char* msg)

source/scriptinterface/ScriptInterface.h

@@ -237,8 +237,13 @@ public:
 	template<typename CHAR> bool Eval(const CHAR* code, JS::MutableHandleValue out);
 	template<typename T, typename CHAR> bool Eval(const CHAR* code, T& out);
 
-	// We have to use a mutable handle because JS_Stringify requires that for unknown reasons.
-	std::wstring ToString(JS::MutableHandleValue obj, bool pretty = false);
+	/**
+	 * Convert an object to a UTF-8 encoded string, either with JSON
+	 * (if pretty == true and there is no JSON error) or with toSource().
+	 *
+	 * We have to use a mutable handle because JS_Stringify requires that for unknown reasons.
+	 */
+	std::string ToString(JS::MutableHandleValue obj, bool pretty = false);
 
 	/**
 	 * Parse a UTF-8-encoded JSON string. Returns the unmodified value on error

source/scriptinterface/tests/test_ScriptInterface.h

@@ -252,6 +252,6 @@ public:
 		TS_ASSERT_STR_EQUALS(stringified, "{\n  \"x\": 1,\n  \"z\": [\n    2,\n    \"3\xE2\x98\xBA\xEF\xBF\xBD\"\n  ],\n  \"y\": true\n}");
 
 		TS_ASSERT(script.ParseJSON(stringified, &val));
-		TS_ASSERT_WSTR_EQUALS(script.ToString(&val), L"({x:1, z:[2, \"3\\u263A\\uFFFD\"], y:true})");
+		TS_ASSERT_STR_EQUALS(script.ToString(&val), "({x:1, z:[2, \"3\\u263A\\uFFFD\"], y:true})");
 	}
 };

source/simulation2/serialization/DebugSerializer.cpp

@@ -149,9 +149,9 @@ void CDebugSerializer::PutString(const char* name, const std::string& value)
 
 void CDebugSerializer::PutScriptVal(const char* name, JS::MutableHandleValue value)
 {
-	std::wstring source = m_ScriptInterface.ToString(value, true);
+	std::string source = m_ScriptInterface.ToString(value, true);
 
-	m_Stream << INDENT << name << ": " << utf8_from_wstring(source) << "\n";
+	m_Stream << INDENT << name << ": " << source << "\n";
 }
 
 void CDebugSerializer::PutRaw(const char* name, const u8* data, size_t len)

source/simulation2/tests/test_CmpTemplateManager.h

@@ -107,19 +107,19 @@ public:
 		const CParamNode* inherit1 = tempMan->LoadTemplate(ent2, "inherit1", -1);
 		JS::RootedValue val(cx);
 		ScriptInterface::ToJSVal(cx, &val, inherit1);
-		TS_ASSERT_WSTR_EQUALS(man.GetScriptInterface().ToString(&val), L"({Test1A:{'@a':\"a1\", '@b':\"b1\", '@c':\"c1\", d:\"d1\", e:\"e1\", f:\"f1\"}})");
+		TS_ASSERT_STR_EQUALS(man.GetScriptInterface().ToString(&val), "({Test1A:{'@a':\"a1\", '@b':\"b1\", '@c':\"c1\", d:\"d1\", e:\"e1\", f:\"f1\"}})");
 
 		const CParamNode* inherit2 = tempMan->LoadTemplate(ent2, "inherit2", -1);
 		ScriptInterface::ToJSVal(cx, &val, inherit2);
-		TS_ASSERT_WSTR_EQUALS(man.GetScriptInterface().ToString(&val), L"({'@parent':\"inherit1\", Test1A:{'@a':\"a2\", '@b':\"b1\", '@c':\"c1\", d:\"d2\", e:\"e1\", f:\"f1\", g:\"g2\"}})");
+		TS_ASSERT_STR_EQUALS(man.GetScriptInterface().ToString(&val), "({'@parent':\"inherit1\", Test1A:{'@a':\"a2\", '@b':\"b1\", '@c':\"c1\", d:\"d2\", e:\"e1\", f:\"f1\", g:\"g2\"}})");
 
 		const CParamNode* actor = tempMan->LoadTemplate(ent2, "actor|example1", -1);
 		ScriptInterface::ToJSVal(cx, &val, &actor->GetChild("VisualActor"));
-		TS_ASSERT_WSTR_EQUALS(man.GetScriptInterface().ToString(&val), L"({Actor:\"example1\", ActorOnly:(void 0), SilhouetteDisplay:\"false\", SilhouetteOccluder:\"false\", VisibleInAtlasOnly:\"false\"})");
+		TS_ASSERT_STR_EQUALS(man.GetScriptInterface().ToString(&val), "({Actor:\"example1\", ActorOnly:(void 0), SilhouetteDisplay:\"false\", SilhouetteOccluder:\"false\", VisibleInAtlasOnly:\"false\"})");
 
 		const CParamNode* foundation = tempMan->LoadTemplate(ent2, "foundation|actor|example1", -1);
 		ScriptInterface::ToJSVal(cx, &val, &foundation->GetChild("VisualActor"));
-		TS_ASSERT_WSTR_EQUALS(man.GetScriptInterface().ToString(&val), L"({Actor:\"example1\", ActorOnly:(void 0), Foundation:(void 0), SilhouetteDisplay:\"false\", SilhouetteOccluder:\"false\", VisibleInAtlasOnly:\"false\"})");
+		TS_ASSERT_STR_EQUALS(man.GetScriptInterface().ToString(&val), "({Actor:\"example1\", ActorOnly:(void 0), Foundation:(void 0), SilhouetteDisplay:\"false\", SilhouetteOccluder:\"false\", VisibleInAtlasOnly:\"false\"})");
 	}
 
 	void test_LoadTemplate_errors()