From b55b236379a80ca402600ddf7f787f827229bdea Mon Sep 17 00:00:00 2001 From: wraitii Date: Wed, 31 Mar 2021 15:55:19 +0000 Subject: [PATCH] Remember OOS on a per-client basis. Change the OOS notification logic to remember the OOS-ness of each client. Reset it on client leave. The server will thus continue checking for OOS if the OOS-client leaves. This is convenient to ignore observer OOS, or wait for an OOS player without restarting the game. Also add the turn number to the OOS dump, to fix #3348: particularly following d4c2cf4430 the turn is likely to not be the same between different clients. Agree by: asterix Differential Revision: https://code.wildfiregames.com/D3753 This was SVN commit r25170. --- source/network/NetClientTurnManager.cpp | 12 +--- source/network/NetServerTurnManager.cpp | 58 +++++++++++-------- source/network/NetServerTurnManager.h | 29 +++++----- source/simulation2/Simulation2.cpp | 1 + source/simulation2/system/ReplayTurnManager.h | 4 +- source/simulation2/system/TurnManager.cpp | 2 +- source/simulation2/system/TurnManager.h | 2 - 7 files changed, 56 insertions(+), 52 deletions(-) diff --git a/source/network/NetClientTurnManager.cpp b/source/network/NetClientTurnManager.cpp index 02f5878b5d..0ab67f4591 100644 --- a/source/network/NetClientTurnManager.cpp +++ b/source/network/NetClientTurnManager.cpp @@ -84,10 +84,6 @@ void CNetClientTurnManager::NotifyFinishedUpdate(u32 turn) m_Replay.Hash(hash, quick); - // Don't send the hash if OOS - if (m_HasSyncError) - return; - // Send message to the server CSyncCheckMessage msg; msg.m_Turn = turn; @@ -114,17 +110,13 @@ void CNetClientTurnManager::OnSyncError(u32 turn, const CStr& expectedHash, cons CStr expectedHashHex(Hexify(expectedHash)); NETCLIENTTURN_LOG("OnSyncError(%d, %hs)\n", turn, expectedHashHex.c_str()); - // Only complain the first time - if (m_HasSyncError) - return; - - m_HasSyncError = true; - std::string hash; ENSURE(m_Simulation2.ComputeStateHash(hash, !TurnNeedsFullHash(turn))); OsPath oosdumpPath(psLogDir() / (L"oos_dump" + g_UniqueLogPostfix + L".txt")); std::ofstream file (OsString(oosdumpPath).c_str(), std::ofstream::out | std::ofstream::trunc); + file << "oos turn: " << turn << std::endl; + file << "net client turn: " << m_CurrentTurn << std::endl; m_Simulation2.DumpDebugState(file); file.close(); diff --git a/source/network/NetServerTurnManager.cpp b/source/network/NetServerTurnManager.cpp index fa2de8260e..350aced5ab 100644 --- a/source/network/NetServerTurnManager.cpp +++ b/source/network/NetServerTurnManager.cpp @@ -35,7 +35,7 @@ #endif CNetServerTurnManager::CNetServerTurnManager(CNetServerWorker& server) - : m_NetServer(server), m_ReadyTurn(COMMAND_DELAY_MP - 1), m_TurnLength(DEFAULT_TURN_LENGTH), m_HasSyncError(false) + : m_NetServer(server), m_ReadyTurn(COMMAND_DELAY_MP - 1), m_TurnLength(DEFAULT_TURN_LENGTH) { // Turn 0 is not actually executed, store a dummy value. m_SavedTurnLengths.push_back(0); @@ -52,21 +52,21 @@ void CNetServerTurnManager::NotifyFinishedClientCommands(CNetServerSession& sess NETSERVERTURN_LOG("NotifyFinishedClientCommands(client=%d, turn=%d)\n", client, turn); // Must be a client we've already heard of - ENSURE(m_ClientsReady.find(client) != m_ClientsReady.end()); + ENSURE(m_ClientsData.find(client) != m_ClientsData.end()); // Clients must advance one turn at a time - if (turn != m_ClientsReady[client] + 1) + if (turn != m_ClientsData[client].readyTurn + 1) { LOGERROR("NotifyFinishedClientCommands: Client %d (%s) is ready for turn %d, but expected %d", client, utf8_from_wstring(session.GetUserName()).c_str(), turn, - m_ClientsReady[client] + 1); + m_ClientsData[client].readyTurn + 1); session.Disconnect(NDR_INCORRECT_READY_TURN_COMMANDS); } - m_ClientsReady[client] = turn; + m_ClientsData[client].readyTurn = turn; // Check whether this was the final client to become ready CheckClientsReady(); @@ -80,13 +80,13 @@ void CNetServerTurnManager::CheckClientsReady() max_observer_lag = max_observer_lag < 0 ? -1 : max_observer_lag > 10000 ? -1 : max_observer_lag; // See if all clients (including self) are ready for a new turn - for (const std::pair& clientReady : m_ClientsReady) + for (const std::pair& clientData : m_ClientsData) { // Observers are allowed to lag more than regular clients. - if (m_ClientsObserver[clientReady.first] && (max_observer_lag == -1 || clientReady.second > m_ReadyTurn - max_observer_lag)) + if (clientData.second.isObserver && (max_observer_lag == -1 || clientData.second.readyTurn > m_ReadyTurn - max_observer_lag)) continue; NETSERVERTURN_LOG(" %d: %d <=? %d\n", clientReady.first, clientReady.second, m_ReadyTurn); - if (clientReady.second <= m_ReadyTurn) + if (clientData.second.readyTurn <= m_ReadyTurn) return; // wasn't ready for m_ReadyTurn+1 } @@ -111,31 +111,31 @@ void CNetServerTurnManager::NotifyFinishedClientUpdate(CNetServerSession& sessio const CStrW& playername = session.GetUserName(); // Clients must advance one turn at a time - if (turn != m_ClientsSimulated[client] + 1) + if (turn != m_ClientsData[client].simulatedTurn + 1) { LOGERROR("NotifyFinishedClientUpdate: Client %d (%s) is ready for turn %d, but expected %d", client, utf8_from_wstring(playername).c_str(), turn, - m_ClientsReady[client] + 1); + m_ClientsData[client].simulatedTurn + 1); session.Disconnect(NDR_INCORRECT_READY_TURN_SIMULATED); } - m_ClientsSimulated[client] = turn; + m_ClientsData[client].simulatedTurn = turn; // Check for OOS only if in sync if (m_HasSyncError) return; - m_ClientPlayernames[client] = playername; + m_ClientsData[client].playerName = playername; m_ClientStateHashes[turn][client] = hash; // Find the newest turn which we know all clients have simulated u32 newest = std::numeric_limits::max(); - for (const std::pair& clientSimulated : m_ClientsSimulated) - if (clientSimulated.second < newest) - newest = clientSimulated.second; + for (const std::pair& clientData : m_ClientsData) + if (clientData.second.simulatedTurn < newest) + newest = clientData.second.simulatedTurn; // For every set of state hashes that all clients have simulated, check for OOS for (const std::pair>& clientStateHash : m_ClientStateHashes) @@ -155,7 +155,8 @@ void CNetServerTurnManager::NotifyFinishedClientUpdate(CNetServerSession& sessio { // Oh no, out of sync m_HasSyncError = true; - OOSPlayerNames.push_back(m_ClientPlayernames[hashPair.first]); + m_ClientsData[hashPair.first].isOOS = true; + OOSPlayerNames.push_back(m_ClientsData[hashPair.first].playerName); } } @@ -184,24 +185,33 @@ void CNetServerTurnManager::InitialiseClient(int client, u32 turn, bool observer { NETSERVERTURN_LOG("InitialiseClient(client=%d, turn=%d)\n", client, turn); - ENSURE(m_ClientsReady.find(client) == m_ClientsReady.end()); - m_ClientsReady[client] = turn + COMMAND_DELAY_MP - 1; - m_ClientsSimulated[client] = turn; - m_ClientsObserver[client] = observer; + ENSURE(m_ClientsData.find(client) == m_ClientsData.end()); + Client& data = m_ClientsData[client]; + data.readyTurn = turn + COMMAND_DELAY_MP - 1; + data.simulatedTurn = turn; + data.isObserver = observer; } void CNetServerTurnManager::UninitialiseClient(int client) { NETSERVERTURN_LOG("UninitialiseClient(client=%d)\n", client); - ENSURE(m_ClientsReady.find(client) != m_ClientsReady.end()); - m_ClientsReady.erase(client); - m_ClientsSimulated.erase(client); - m_ClientsObserver.erase(client); + ENSURE(m_ClientsData.find(client) != m_ClientsData.end()); + bool checkOOS = m_ClientsData[client].isOOS; + m_ClientsData.erase(client); // Check whether we're ready for the next turn now that we're not // waiting for this client any more CheckClientsReady(); + + // Check whether we're still OOS. + if (checkOOS) + { + for (const std::pair& clientData : m_ClientsData) + if (clientData.second.isOOS) + return; + m_HasSyncError = false; + } } void CNetServerTurnManager::SetTurnLength(u32 msecs) diff --git a/source/network/NetServerTurnManager.h b/source/network/NetServerTurnManager.h index 1ac71ec7eb..5133cdf82b 100644 --- a/source/network/NetServerTurnManager.h +++ b/source/network/NetServerTurnManager.h @@ -72,24 +72,27 @@ public: private: void CheckClientsReady(); - /// The latest turn for which we have received all commands from all clients - u32 m_ReadyTurn; + struct Client + { + CStrW playerName; + // Latest turn for which all commands have been received. + u32 readyTurn; + // Last known simulated turn. + u32 simulatedTurn; + bool isObserver; + bool isOOS = false; + }; - // Client ID -> whether they are only an observer. - std::unordered_map m_ClientsObserver; + std::unordered_map m_ClientsData; - // Client ID -> ready turn number (the latest turn for which all commands have been received from that client) - std::map m_ClientsReady; - - // Client ID -> last known simulated turn number (for which we have the state hash) - // (the client has reached the start of this turn, not done the update for it yet) - std::map m_ClientsSimulated; + // Cached value - is any client OOS? This is reset when the OOS client leaves. + bool m_HasSyncError = false; // Map of turn -> {Client ID -> state hash}; old indexes <= min(m_ClientsSimulated) are deleted std::map> m_ClientStateHashes; - // Map of client ID -> playername - std::map m_ClientPlayernames; + /// The latest turn for which we have received all commands from all clients + u32 m_ReadyTurn; // Current turn length u32 m_TurnLength; @@ -98,8 +101,6 @@ private: std::vector m_SavedTurnLengths; CNetServerWorker& m_NetServer; - - bool m_HasSyncError; }; #endif // INCLUDED_NETSERVERTURNMANAGER diff --git a/source/simulation2/Simulation2.cpp b/source/simulation2/Simulation2.cpp index bb250da246..4dc509334f 100644 --- a/source/simulation2/Simulation2.cpp +++ b/source/simulation2/Simulation2.cpp @@ -885,6 +885,7 @@ bool CSimulation2::ComputeStateHash(std::string& outHash, bool quick) bool CSimulation2::DumpDebugState(std::ostream& stream) { + stream << "sim turn: " << m->m_TurnNumber << std::endl; return m->m_ComponentManager.DumpDebugState(stream, true); } diff --git a/source/simulation2/system/ReplayTurnManager.h b/source/simulation2/system/ReplayTurnManager.h index d1b1ce394e..67d52a84f8 100644 --- a/source/simulation2/system/ReplayTurnManager.h +++ b/source/simulation2/system/ReplayTurnManager.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2020 Wildfire Games. +/* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -44,6 +44,8 @@ private: static const CStr EventNameReplayFinished; static const CStr EventNameReplayOutOfSync; + bool m_HasSyncError = false; + // Contains the commands of every player on each turn std::map>> m_ReplayCommands; diff --git a/source/simulation2/system/TurnManager.cpp b/source/simulation2/system/TurnManager.cpp index 100a960cd6..b068468223 100644 --- a/source/simulation2/system/TurnManager.cpp +++ b/source/simulation2/system/TurnManager.cpp @@ -40,7 +40,7 @@ const CStr CTurnManager::EventNameSavegameLoaded = "SavegameLoaded"; CTurnManager::CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, u32 commandDelay, int clientId, IReplayLogger& replay) : m_Simulation2(simulation), m_CurrentTurn(0), m_CommandDelay(commandDelay), m_ReadyTurn(commandDelay - 1), m_TurnLength(defaultTurnLength), - m_PlayerId(-1), m_ClientId(clientId), m_DeltaSimTime(0), m_HasSyncError(false), m_Replay(replay), + m_PlayerId(-1), m_ClientId(clientId), m_DeltaSimTime(0), m_Replay(replay), m_FinalTurn(std::numeric_limits::max()), m_TimeWarpNumTurns(0), m_QuickSaveMetadata(m_Simulation2.GetScriptInterface().GetGeneralJSContext()) { diff --git a/source/simulation2/system/TurnManager.h b/source/simulation2/system/TurnManager.h index 588d57e8cb..3433f52832 100644 --- a/source/simulation2/system/TurnManager.h +++ b/source/simulation2/system/TurnManager.h @@ -209,8 +209,6 @@ protected: /// add elapsed time increments to until we reach 0). float m_DeltaSimTime; - bool m_HasSyncError; - IReplayLogger& m_Replay; // The number of the last turn that is allowed to be executed (used for replays)