From e908733220327f39283b66f8d70187f14c305fc3 Mon Sep 17 00:00:00 2001 From: wraitii Date: Sun, 16 May 2021 13:50:05 +0000 Subject: [PATCH] Speed up GetAvailableMods for archives. Refs 64bfa089af and 44ec2e324e When a .zip file is encountered by the VFS population, it reads the info for all files in the archives. This is quite slow for the public archive (400-500ms on my computer), which means calling GetEngineInfo() repeatedly is impossible. By only opening the external mod.json, we skip most of the work. The archive can still be opened if needed as fallback. Differential Revision: https://code.wildfiregames.com/D3216 This was SVN commit r25446. --- source/lib/sysdep/os/osx/osx_atlas.mm | 12 +- source/main.cpp | 6 +- source/ps/GameSetup/CmdLineArgs.cpp | 4 +- source/ps/GameSetup/GameSetup.cpp | 2 +- source/ps/Mod.cpp | 179 +++++++++++++++----------- source/ps/Mod.h | 7 +- source/ps/ModInstaller.cpp | 11 +- source/ps/ModInstaller.h | 3 +- source/ps/ModIo.cpp | 4 +- source/ps/VisualReplay.cpp | 2 +- 10 files changed, 134 insertions(+), 96 deletions(-) diff --git a/source/lib/sysdep/os/osx/osx_atlas.mm b/source/lib/sysdep/os/osx/osx_atlas.mm index ccd7c405a9..6254318724 100644 --- a/source/lib/sysdep/os/osx/osx_atlas.mm +++ b/source/lib/sysdep/os/osx/osx_atlas.mm @@ -1,4 +1,4 @@ -/* Copyright (C) 2020 Wildfire Games. +/* Copyright (C) 2021 Wildfire Games. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -25,11 +25,15 @@ #import "osx_atlas.h" -#include #include "lib/types.h" #include "ps/CStr.h" -extern std::vector g_modsLoaded; +#include + +namespace Mod +{ +extern std::vector g_ModsLoaded; +} void startNewAtlasProcess() { @@ -39,7 +43,7 @@ void startNewAtlasProcess() [args addObject:@"--editor"]; // Pass mods on the command line. - for (const CStr& mod : g_modsLoaded) + for (const CStr& mod : Mod::g_ModsLoaded) { std::string arg = std::string("-mod=") + mod; [args addObject:[[NSString alloc] initWithUTF8String:arg.c_str()]]; diff --git a/source/main.cpp b/source/main.cpp index e89a71a8ee..eaa06f2cb4 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -99,7 +99,7 @@ that of Atlas depending on commandline parameters. #include -extern CmdLineArgs g_args; +extern CmdLineArgs g_CmdLineArgs; extern CStrW g_UniqueLogPostfix; // Marks terrain as modified so the minimap can repaint (is there a cleaner way of handling this?) @@ -183,7 +183,7 @@ static InReaction MainInputHandler(const SDL_Event_* ev) case SDL_DROPFILE: { char* dropped_filedir = ev->ev.drop.file; - const Paths paths(g_args); + const Paths paths(g_CmdLineArgs); CModInstaller installer(paths.UserData() / "mods", paths.Cache()); installer.Install(std::string(dropped_filedir), g_ScriptContext, true); SDL_free(dropped_filedir); @@ -512,7 +512,7 @@ static void RunGameOrAtlas(int argc, const char* argv[]) { CmdLineArgs args(argc, argv); - g_args = args; + g_CmdLineArgs = args; if (args.Has("version")) { diff --git a/source/ps/GameSetup/CmdLineArgs.cpp b/source/ps/GameSetup/CmdLineArgs.cpp index a244375b8d..5d6a17f777 100644 --- a/source/ps/GameSetup/CmdLineArgs.cpp +++ b/source/ps/GameSetup/CmdLineArgs.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2018 Wildfire Games. +/* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -20,6 +20,8 @@ #include "lib/sysdep/sysdep.h" +CmdLineArgs g_CmdLineArgs; + namespace { diff --git a/source/ps/GameSetup/GameSetup.cpp b/source/ps/GameSetup/GameSetup.cpp index 288b943274..c0d8b8a258 100644 --- a/source/ps/GameSetup/GameSetup.cpp +++ b/source/ps/GameSetup/GameSetup.cpp @@ -945,7 +945,7 @@ bool Init(const CmdLineArgs& args, int flags) return false; } } - else if (!EnableModsOrSetDefault(args, g_modsLoaded, false)) + else if (!EnableModsOrSetDefault(args, Mod::g_ModsLoaded, false)) return false; } diff --git a/source/ps/Mod.cpp b/source/ps/Mod.cpp index fea1542e1a..9e4f3c1c26 100644 --- a/source/ps/Mod.cpp +++ b/source/ps/Mod.cpp @@ -19,12 +19,14 @@ #include "ps/Mod.h" +#include "i18n/L10n.h" #include "lib/file/file_system.h" #include "lib/file/vfs/vfs.h" #include "lib/utf8.h" #include "ps/Filesystem.h" #include "ps/GameSetup/GameSetup.h" #include "ps/GameSetup/Paths.h" +#include "ps/Profiler2.h" #include "ps/Pyrogenesis.h" #include "scriptinterface/Object.h" #include "scriptinterface/ScriptInterface.h" @@ -33,22 +35,71 @@ #include #include #include +#include +#include #include -std::vector g_modsLoaded; -std::vector g_incompatibleMods; -std::vector g_failedMods; +namespace Mod +{ +std::vector g_ModsLoaded; +std::vector g_IncompatibleMods; +std::vector g_FailedMods; std::vector> g_LoadedModVersions; -CmdLineArgs g_args; - -JS::Value Mod::GetAvailableMods(const ScriptInterface& scriptInterface) +bool ParseModJSON(const ScriptRequest& rq, const PIVFS& vfs, OsPath modsPath, OsPath mod, JS::MutableHandleValue json) { - ScriptRequest rq(scriptInterface); - JS::RootedObject obj(rq.cx, JS_NewPlainObject(rq.cx)); + // Attempt to open mod.json first. + std::ifstream modjson; + modjson.open((modsPath / mod / L"mod.json").string8()); - const Paths paths(g_args); + if (!modjson.is_open()) + { + modjson.close(); + + // Fallback: open the archive and read mod.json there. + // This can take in the hundreds of milliseconds with large mods. + vfs->Clear(); + if (vfs->Mount(L"", modsPath / mod / "", VFS_MOUNT_MUST_EXIST, VFS_MIN_PRIORITY) < 0) + return false; + + CVFSFile modinfo; + if (modinfo.Load(vfs, L"mod.json", false) != PSRETURN_OK) + return false; + + if (!Script::ParseJSON(rq, modinfo.GetAsString(), json)) + return false; + + // Attempt to write the mod.json file so we'll take the fast path next time. + std::ofstream out_mod_json((modsPath / mod / L"mod.json").string8()); + if (out_mod_json.good()) + { + out_mod_json << modinfo.GetAsString(); + out_mod_json.close(); + } + else + { + // Print a warning - we'll keep trying, which could have adverse effects. + if (L10n::IsInitialised()) + LOGWARNING(g_L10n.Translate("Could not write external mod.json for zipped mod '%s'. The mod should be reinstalled."), mod.string8()); + else + LOGWARNING("Could not write external mod.json for zipped mod '%s'. The mod should be reinstalled.", mod.string8()); + } + return true; + } + else + { + std::stringstream buffer; + buffer << modjson.rdbuf(); + return Script::ParseJSON(rq, buffer.str(), json); + } +} + +JS::Value GetAvailableMods(const ScriptInterface& scriptInterface) +{ + PROFILE2("GetAvailableMods"); + + const Paths paths(g_CmdLineArgs); // loop over all possible paths OsPath modPath = paths.RData()/"mods"; @@ -63,115 +114,96 @@ JS::Value Mod::GetAvailableMods(const ScriptInterface& scriptInterface) PIVFS vfs = CreateVfs(); + ScriptRequest rq(scriptInterface); + JS::RootedValue value(rq.cx, Script::CreateObject(rq)); + for (DirectoryNames::iterator iter = modDirs.begin(); iter != modDirs.end(); ++iter) { - vfs->Clear(); - // Mount with lowest priority, we don't want to overwrite anything - if (vfs->Mount(L"", modPath / *iter / "", VFS_MOUNT_MUST_EXIST, VFS_MIN_PRIORITY) < 0) - continue; - - CVFSFile modinfo; - if (modinfo.Load(vfs, L"mod.json", false) != PSRETURN_OK) - continue; - JS::RootedValue json(rq.cx); - if (!Script::ParseJSON(rq, modinfo.GetAsString(), &json)) + if (!ParseModJSON(rq, vfs, modPath, *iter, &json)) continue; - - // Valid mod, add it to our structure - JS_SetProperty(rq.cx, obj, utf8_from_wstring(iter->string()).c_str(), json); + // Valid mod data, add it to our structure + Script::SetProperty(rq, value, utf8_from_wstring(iter->string()).c_str(), json); } GetDirectoryEntries(modUserPath, NULL, &modDirsUser); - bool dev = InDevelopmentCopy(); for (DirectoryNames::iterator iter = modDirsUser.begin(); iter != modDirsUser.end(); ++iter) { - // If we are in a dev copy we do not mount mods in the user mod folder that - // are already present in the mod folder, thus we skip those here. - if (dev && std::binary_search(modDirs.begin(), modDirs.end(), *iter)) - continue; - - vfs->Clear(); - // Mount with lowest priority, we don't want to overwrite anything - if (vfs->Mount(L"", modUserPath / *iter / "", VFS_MOUNT_MUST_EXIST, VFS_MIN_PRIORITY) < 0) - continue; - - CVFSFile modinfo; - if (modinfo.Load(vfs, L"mod.json", false) != PSRETURN_OK) + // Ignore mods in the user folder if we have already found them in modDirs. + if (std::binary_search(modDirs.begin(), modDirs.end(), *iter)) continue; JS::RootedValue json(rq.cx); - if (!Script::ParseJSON(rq, modinfo.GetAsString(), &json)) + if (!ParseModJSON(rq, vfs, modUserPath, *iter, &json)) continue; - - // Valid mod, add it to our structure - JS_SetProperty(rq.cx, obj, utf8_from_wstring(iter->string()).c_str(), json); + // Valid mod data, add it to our structure + Script::SetProperty(rq, value, utf8_from_wstring(iter->string()).c_str(), json); } - return JS::ObjectValue(*obj); + return value.get(); } -const std::vector& Mod::GetEnabledMods() +const std::vector& GetEnabledMods() { - return g_modsLoaded; + return g_ModsLoaded; } -const std::vector& Mod::GetIncompatibleMods() +const std::vector& GetIncompatibleMods() { - return g_incompatibleMods; + return g_IncompatibleMods; } -const std::vector& Mod::GetFailedMods() +const std::vector& GetFailedMods() { - return g_failedMods; + return g_FailedMods; } -const std::vector& Mod::GetModsFromArguments(const CmdLineArgs& args, int flags) +const std::vector& GetModsFromArguments(const CmdLineArgs& args, int flags) { const bool initMods = (flags & INIT_MODS) == INIT_MODS; const bool addPublic = (flags & INIT_MODS_PUBLIC) == INIT_MODS_PUBLIC; if (!initMods) - return g_modsLoaded; + return g_ModsLoaded; - g_modsLoaded = args.GetMultiple("mod"); + g_ModsLoaded = args.GetMultiple("mod"); if (addPublic) - g_modsLoaded.insert(g_modsLoaded.begin(), "public"); + g_ModsLoaded.insert(g_ModsLoaded.begin(), "public"); - g_modsLoaded.insert(g_modsLoaded.begin(), "mod"); + g_ModsLoaded.insert(g_ModsLoaded.begin(), "mod"); - return g_modsLoaded; + return g_ModsLoaded; } -void Mod::SetDefaultMods() +void SetDefaultMods() { - g_modsLoaded.clear(); - g_modsLoaded.insert(g_modsLoaded.begin(), "mod"); + g_ModsLoaded.clear(); + g_ModsLoaded.insert(g_ModsLoaded.begin(), "mod"); } -void Mod::ClearIncompatibleMods() +void ClearIncompatibleMods() { - g_incompatibleMods.clear(); - g_failedMods.clear(); + g_IncompatibleMods.clear(); + g_FailedMods.clear(); } -bool Mod::CheckAndEnableMods(const ScriptInterface& scriptInterface, const std::vector& mods) +bool CheckAndEnableMods(const ScriptInterface& scriptInterface, const std::vector& mods) { ScriptRequest rq(scriptInterface); JS::RootedValue availableMods(rq.cx, GetAvailableMods(scriptInterface)); if (!AreModsCompatible(scriptInterface, mods, availableMods)) { - g_failedMods = mods; + g_FailedMods = mods; return false; } - g_modsLoaded = mods; + g_ModsLoaded = mods; return true; } -bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::vector& mods, const JS::RootedValue& availableMods) +bool AreModsCompatible(const ScriptInterface& scriptInterface, const std::vector& mods, const JS::RootedValue& availableMods) { ScriptRequest rq(scriptInterface); std::unordered_map> modDependencies; @@ -186,12 +218,12 @@ bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::v // Requested mod is not available, fail if (!Script::HasProperty(rq, availableMods, mod.c_str())) { - g_incompatibleMods.push_back(mod); + g_IncompatibleMods.push_back(mod); continue; } if (!Script::GetProperty(rq, availableMods, mod.c_str(), &modData)) { - g_incompatibleMods.push_back(mod); + g_IncompatibleMods.push_back(mod); continue; } @@ -236,13 +268,13 @@ bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::v const std::unordered_map::iterator it = modNameVersions.find(modToCheck); if (it == modNameVersions.end()) { - g_incompatibleMods.push_back(mod); + g_IncompatibleMods.push_back(mod); continue; } // 0.0.25(0ad) , <=, 0.0.24(required version) if (!CompareVersionStrings(it->second, op, versionToCheck)) { - g_incompatibleMods.push_back(mod); + g_IncompatibleMods.push_back(mod); continue; } break; @@ -251,10 +283,10 @@ bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::v } - return g_incompatibleMods.empty(); + return g_IncompatibleMods.empty(); } -bool Mod::CompareVersionStrings(const CStr& version, const CStr& op, const CStr& required) +bool CompareVersionStrings(const CStr& version, const CStr& op, const CStr& required) { std::vector versionSplit; std::vector requiredSplit; @@ -288,7 +320,7 @@ bool Mod::CompareVersionStrings(const CStr& version, const CStr& op, const CStr& } -void Mod::CacheEnabledModVersions(const shared_ptr& scriptContext) +void CacheEnabledModVersions(const shared_ptr& scriptContext) { ScriptInterface scriptInterface("Engine", "CacheEnabledModVersions", scriptContext); ScriptRequest rq(scriptInterface); @@ -297,7 +329,7 @@ void Mod::CacheEnabledModVersions(const shared_ptr& scriptContext g_LoadedModVersions.clear(); - for (const CStr& mod : g_modsLoaded) + for (const CStr& mod : g_ModsLoaded) { // Ignore mod mod as it is irrelevant for compatibility checks if (mod == "mod") @@ -312,7 +344,7 @@ void Mod::CacheEnabledModVersions(const shared_ptr& scriptContext } } -JS::Value Mod::GetLoadedModsWithVersions(const ScriptInterface& scriptInterface) +JS::Value GetLoadedModsWithVersions(const ScriptInterface& scriptInterface) { ScriptRequest rq(scriptInterface); JS::RootedValue returnValue(rq.cx); @@ -320,11 +352,11 @@ JS::Value Mod::GetLoadedModsWithVersions(const ScriptInterface& scriptInterface) return returnValue; } -JS::Value Mod::GetEngineInfo(const ScriptInterface& scriptInterface) +JS::Value GetEngineInfo(const ScriptInterface& scriptInterface) { ScriptRequest rq(scriptInterface); - JS::RootedValue mods(rq.cx, Mod::GetLoadedModsWithVersions(scriptInterface)); + JS::RootedValue mods(rq.cx, GetLoadedModsWithVersions(scriptInterface)); JS::RootedValue metainfo(rq.cx); Script::CreateObject( @@ -337,3 +369,4 @@ JS::Value Mod::GetEngineInfo(const ScriptInterface& scriptInterface) return metainfo; } +} diff --git a/source/ps/Mod.h b/source/ps/Mod.h index ebf40d495e..28150807e3 100644 --- a/source/ps/Mod.h +++ b/source/ps/Mod.h @@ -22,11 +22,14 @@ #include "ps/GameSetup/CmdLineArgs.h" #include "scriptinterface/ScriptForward.h" -extern std::vector g_modsLoaded; -extern CmdLineArgs g_args; +#include + +extern CmdLineArgs g_CmdLineArgs; namespace Mod { + extern std::vector g_ModsLoaded; + JS::Value GetAvailableMods(const ScriptInterface& scriptInterface); const std::vector& GetEnabledMods(); const std::vector& GetIncompatibleMods(); diff --git a/source/ps/ModInstaller.cpp b/source/ps/ModInstaller.cpp index 16615c582b..5457fcd24f 100644 --- a/source/ps/ModInstaller.cpp +++ b/source/ps/ModInstaller.cpp @@ -25,9 +25,7 @@ #include "scriptinterface/ScriptInterface.h" #include "scriptinterface/JSON.h" -#ifdef OS_WIN #include -#endif CModInstaller::CModInstaller(const OsPath& modsdir, const OsPath& tempdir) : m_ModsDir(modsdir), m_TempDir(tempdir / "_modscache"), m_CacheDir("cache/") @@ -88,23 +86,20 @@ CModInstaller::ModInstallationResult CModInstaller::Install( // Create a directory with the following structure: // mod-name/ // mod-name.zip + // mod.json CreateDirectories(modDir, 0700); if (wrename(modTemp, modPath) != 0) return FAIL_ON_MOD_MOVE; DeleteDirectory(modTemp.Parent()); -#ifdef OS_WIN - // On Windows, write the contents of mod.json to a separate file next to the archive: - // mod-name/ - // mod-name.zip - // mod.json std::ofstream mod_json((modDir / "mod.json").string8()); if (mod_json.good()) { mod_json << modinfo.GetAsString(); mod_json.close(); } -#endif // OS_WIN + else + return FAIL_ON_JSON_WRITE; m_InstalledMods.emplace_back(modName); diff --git a/source/ps/ModInstaller.h b/source/ps/ModInstaller.h index e5980f3cdf..3121034f10 100644 --- a/source/ps/ModInstaller.h +++ b/source/ps/ModInstaller.h @@ -38,7 +38,8 @@ public: FAIL_ON_MOD_LOAD, FAIL_ON_PARSE_JSON, FAIL_ON_EXTRACT_NAME, - FAIL_ON_MOD_MOVE + FAIL_ON_MOD_MOVE, + FAIL_ON_JSON_WRITE }; /** diff --git a/source/ps/ModIo.cpp b/source/ps/ModIo.cpp index 31b8c4cf5f..a381c094df 100644 --- a/source/ps/ModIo.cpp +++ b/source/ps/ModIo.cpp @@ -309,7 +309,7 @@ void ModIo::StartDownloadMod(u32 idx) if (idx >= m_ModData.size()) return; - const Paths paths(g_args); + const Paths paths(g_CmdLineArgs); const OsPath modUserPath = paths.UserData()/"mods"; const OsPath modPath = modUserPath/m_ModData[idx].properties["name_id"]; if (!DirectoryExists(modPath) && INFO::OK != CreateDirectories(modPath, 0700, false)) @@ -482,7 +482,7 @@ bool ModIo::AdvanceRequest(const ScriptInterface& scriptInterface) m_DownloadProgressData.status = DownloadProgressStatus::SUCCESS; { - Paths paths(g_args); + Paths paths(g_CmdLineArgs); CModInstaller installer(paths.UserData() / "mods", paths.Cache()); installer.Install(m_DownloadFilePath, g_ScriptContext, false); } diff --git a/source/ps/VisualReplay.cpp b/source/ps/VisualReplay.cpp index 1e6ed85338..42cdbb2308 100644 --- a/source/ps/VisualReplay.cpp +++ b/source/ps/VisualReplay.cpp @@ -44,7 +44,7 @@ const u8 minimumReplayDuration = 3; OsPath VisualReplay::GetDirectoryPath() { - return Paths(g_args).UserData() / "replays" / engine_version; + return Paths(g_CmdLineArgs).UserData() / "replays" / engine_version; } OsPath VisualReplay::GetCacheFilePath()