From 0d172264f811a4e37bc4055ef85aab77ff42ee99 Mon Sep 17 00:00:00 2001 From: Ykkrosh Date: Sat, 18 Sep 2010 18:21:00 +0000 Subject: [PATCH] # Add -archivebuild mode to generate .zip files for releases, with automatic compression of textures. Fix terrain manager to understand .cached.dds files. Fix IArchiveWriter so you can pass it absolute paths of files to load, and different relative paths for storing inside the archive. Fix fs_util::ForEachFile when called on the VFS root. This was SVN commit r8130. --- source/graphics/TerrainTextureManager.cpp | 15 +++ source/graphics/TextureManager.cpp | 36 ++++++- source/graphics/TextureManager.h | 8 ++ source/lib/file/archive/archive.h | 5 +- source/lib/file/archive/archive_zip.cpp | 6 +- source/lib/file/file_system_util.cpp | 10 +- source/main.cpp | 21 +++++ source/ps/ArchiveBuilder.cpp | 110 ++++++++++++++++++++++ source/ps/ArchiveBuilder.h | 64 +++++++++++++ source/ps/Filesystem.cpp | 7 +- source/ps/Filesystem.h | 2 + source/ps/XML/Xeromyces.cpp | 4 +- 12 files changed, 279 insertions(+), 9 deletions(-) create mode 100644 source/ps/ArchiveBuilder.cpp create mode 100644 source/ps/ArchiveBuilder.h diff --git a/source/graphics/TerrainTextureManager.cpp b/source/graphics/TerrainTextureManager.cpp index 56247e1005..7021100713 100644 --- a/source/graphics/TerrainTextureManager.cpp +++ b/source/graphics/TerrainTextureManager.cpp @@ -31,6 +31,8 @@ #include "ps/CLogger.h" #include "ps/Filesystem.h" +#include + #define LOG_CATEGORY L"graphics" CTerrainTextureManager::CTerrainTextureManager(): @@ -111,6 +113,19 @@ void CTerrainTextureManager::LoadTextures(const CTerrainPropertiesPtr& props, co VfsPaths pathnames; if(fs_util::GetPathnames(g_VFS, path, 0, pathnames) < 0) return; + + // If we have any .cached.dds files then strip that extension to get the + // 'real' texture name + for(size_t i = 0; i < pathnames.size(); i++) + { + if(boost::algorithm::ends_with(pathnames[i].leaf(), L".cached.dds")) + pathnames[i] = pathnames[i].branch_path() / boost::algorithm::erase_last_copy(pathnames[i].leaf(), L".cached.dds"); + } + + // Remove any duplicates created by the stripping + std::sort(pathnames.begin(), pathnames.end()); + pathnames.erase(std::unique(pathnames.begin(), pathnames.end()), pathnames.end()); + for(size_t i = 0; i < pathnames.size(); i++) { // skip files that obviously aren't textures. diff --git a/source/graphics/TextureManager.cpp b/source/graphics/TextureManager.cpp index 2848ddc517..b09280e6a1 100644 --- a/source/graphics/TextureManager.cpp +++ b/source/graphics/TextureManager.cpp @@ -271,7 +271,7 @@ public: VfsPath sourcePath = texture->m_Properties.m_Path; VfsPath sourceDir = sourcePath.branch_path(); std::wstring sourceName = sourcePath.leaf(); - VfsPath archiveCachePath = sourceDir / (sourceName + L".dds"); + VfsPath archiveCachePath = sourceDir / (sourceName + L".cached.dds"); // Try the archive cache file first if (CanUseArchiveCache(sourcePath, archiveCachePath)) @@ -371,6 +371,35 @@ public: m_TextureConverter.ConvertTexture(texture, sourcePath, looseCachePath, settings); } + bool GenerateCachedTexture(const VfsPath& sourcePath, VfsPath& archiveCachePath) + { + VfsPath sourceDir = sourcePath.branch_path(); + std::wstring sourceName = sourcePath.leaf(); + archiveCachePath = sourceDir / (sourceName + L".cached.dds"); + + CTextureProperties textureProps(sourcePath); + CTexturePtr texture = CreateTexture(textureProps); + CTextureConverter::Settings settings = GetConverterSettings(texture); + + // TODO: we ought to use a higher-quality compression mode here + // (since we don't care about performance) + + if (!m_TextureConverter.ConvertTexture(texture, sourcePath, L"cache"/archiveCachePath, settings)) + return false; + + while (true) + { + CTexturePtr textureOut; + VfsPath dest; + bool ok; + if (m_TextureConverter.Poll(textureOut, dest, ok)) + return ok; + + // Spin-loop is dumb but it works okay for now + SDL_Delay(0); + } + } + bool MakeProgress() { // Process any completed conversion tasks @@ -672,3 +701,8 @@ bool CTextureManager::MakeProgress() { return m->MakeProgress(); } + +bool CTextureManager::GenerateCachedTexture(const VfsPath& path, VfsPath& outputPath) +{ + return m->GenerateCachedTexture(path, outputPath); +} diff --git a/source/graphics/TextureManager.h b/source/graphics/TextureManager.h index c314bb8775..49ee51eb44 100644 --- a/source/graphics/TextureManager.h +++ b/source/graphics/TextureManager.h @@ -100,6 +100,14 @@ public: */ bool MakeProgress(); + /** + * Synchronously converts and compresses and saves the texture, + * and returns the output path (minus a "cache/" prefix). This + * is intended for pre-caching textures in release archives. + * @return true on success + */ + bool GenerateCachedTexture(const VfsPath& path, VfsPath& outputPath); + private: CTextureManagerImpl* m; }; diff --git a/source/lib/file/archive/archive.h b/source/lib/file/archive/archive.h index 9e3a009a43..c3b1499369 100644 --- a/source/lib/file/archive/archive.h +++ b/source/lib/file/archive/archive.h @@ -87,8 +87,11 @@ struct IArchiveWriter * archived versions. however, the archive builder usually adds files * precisely because they aren't in archives, and the cache would * thrash anyway, so this is deemed acceptable. + * + * @param sourcepathname the path to the source file on the filesystem + * @param pathname the path to use for the file inside the archive **/ - virtual LibError AddFile(const fs::wpath& pathname) = 0; + virtual LibError AddFile(const fs::wpath& sourcepathname, const fs::wpath& pathame) = 0; }; typedef shared_ptr PIArchiveWriter; diff --git a/source/lib/file/archive/archive_zip.cpp b/source/lib/file/archive/archive_zip.cpp index 33d9d0ba32..4ec7246925 100644 --- a/source/lib/file/archive/archive_zip.cpp +++ b/source/lib/file/archive/archive_zip.cpp @@ -560,10 +560,10 @@ public: wtruncate(pathname.string().c_str(), m_fileSize); } - LibError AddFile(const fs::wpath& pathname) + LibError AddFile(const fs::wpath& sourcepathname, const fs::wpath& pathname) { FileInfo fileInfo; - RETURN_ERR(GetFileInfo(pathname, &fileInfo)); + RETURN_ERR(GetFileInfo(sourcepathname, &fileInfo)); const off_t usize = fileInfo.Size(); // skip 0-length files. // rationale: zip.cpp needs to determine whether a CDFH entry is @@ -577,7 +577,7 @@ public: return INFO::SKIPPED; PFile file(new File); - RETURN_ERR(file->Open(pathname, 'r')); + RETURN_ERR(file->Open(sourcepathname, 'r')); const size_t pathnameLength = pathname.string().length(); diff --git a/source/lib/file/file_system_util.cpp b/source/lib/file/file_system_util.cpp index a3744c3a62..b898f19f23 100644 --- a/source/lib/file/file_system_util.cpp +++ b/source/lib/file/file_system_util.cpp @@ -112,7 +112,15 @@ LibError ForEachFile(const PIVFS& fs, const VfsPath& startPath, FileCallback cb, break; for(size_t i = 0; i < subdirectoryNames.size(); i++) - pendingDirectories.push(AddSlash(path/subdirectoryNames[i])); + { + VfsPath pathname; + if (path.string() == L"/") // special case for startPath == L"" + pathname = AddSlash(VfsPath(subdirectoryNames[i])); + else + pathname = AddSlash(path/subdirectoryNames[i]); + + pendingDirectories.push(pathname); + } pendingDirectories.pop(); } diff --git a/source/main.cpp b/source/main.cpp index 3780e01caa..725eb75c1f 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -38,9 +38,11 @@ that of Atlas depending on commandline parameters. #include "lib/input.h" #include "lib/ogl.h" #include "lib/timer.h" +#include "lib/utf8.h" #include "lib/external_libraries/sdl.h" #include "lib/res/sound/snd_mgr.h" +#include "ps/ArchiveBuilder.h" #include "ps/CConsole.h" #include "ps/Filesystem.h" #include "ps/Game.h" @@ -454,6 +456,25 @@ static void RunGameOrAtlas(int argc, const char* argv[]) return; } + // run in archive-building mode if requested + if (args.Has("archivebuild")) + { + Paths paths(args); + + fs::wpath mod = wstring_from_utf8(args.Get("archivebuild")); + fs::wpath zip; + if (args.Has("archivebuild-output")) + zip = wstring_from_utf8(args.Get("archivebuild-output")); + else + zip = mod.leaf()+L".zip"; + + CArchiveBuilder builder(mod, paths.Cache()); + builder.Build(zip); + + CXeromyces::Terminate(); + return; + } + const double res = timer_Resolution(); g_frequencyFilter = CreateFrequencyFilter(res, 30.0); diff --git a/source/ps/ArchiveBuilder.cpp b/source/ps/ArchiveBuilder.cpp new file mode 100644 index 0000000000..afab11c85d --- /dev/null +++ b/source/ps/ArchiveBuilder.cpp @@ -0,0 +1,110 @@ +/* Copyright (C) 2010 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#include "precompiled.h" + +#include "ArchiveBuilder.h" + +#include "graphics/TextureManager.h" +#include "lib/path_util.h" +#include "lib/tex/tex_codec.h" +#include "lib/file/archive/archive_zip.h" + +#include + +CArchiveBuilder::CArchiveBuilder(const fs::wpath& mod, const fs::wpath& tempdir) : + m_TempDir(tempdir) +{ + tex_codec_register_all(); + + m_VFS = CreateVfs(20*MiB); + + DeleteDirectory(m_TempDir/L"_archivecache"); // clean up in case the last run failed + + m_VFS->Mount(L"cache/", m_TempDir/L"_archivecache/"); + + m_VFS->Mount(L"", AddSlash(mod), VFS_MOUNT_MUST_EXIST); + + // Collect the list of files before loading any base mods + fs_util::ForEachFile(m_VFS, L"", &CollectFileCB, (uintptr_t)static_cast(this), 0, fs_util::DIR_RECURSIVE); +} + +CArchiveBuilder::~CArchiveBuilder() +{ + m_VFS.reset(); + + DeleteDirectory(m_TempDir/L"_archivecache"); + + tex_codec_unregister_all(); +} + +void CArchiveBuilder::AddBaseMod(const fs::wpath& mod) +{ + m_VFS->Mount(L"", AddSlash(mod), VFS_MOUNT_MUST_EXIST); +} + +void CArchiveBuilder::Build(const fs::wpath& archive) +{ + PIArchiveWriter writer = CreateArchiveWriter_Zip(archive); + + // Use CTextureManager instead of CTextureConverter directly, + // so it can deal with all the loading of settings.xml files + CTextureManager texman(m_VFS, true); + + for (size_t i = 0; i < m_Files.size(); ++i) + { + LibError ret; + + fs::wpath realPath; + ret = m_VFS->GetRealPath(m_Files[i], realPath); + debug_assert(ret == INFO::OK); + + // Compress textures and store the new cached version instead of the original + if (boost::algorithm::starts_with(m_Files[i].string(), L"art/textures/") && + tex_is_known_extension(m_Files[i]) && + // Skip some subdirectories where the engine doesn't use CTextureManager yet: + !boost::algorithm::starts_with(m_Files[i].string(), L"art/textures/cursors/") && + !boost::algorithm::starts_with(m_Files[i].string(), L"art/textures/terrain/alphamaps/special/") + ) + { + VfsPath cachedPath; + debug_printf(L"Converting texture %ls\n", realPath.string().c_str()); + bool ok = texman.GenerateCachedTexture(m_Files[i], cachedPath); + debug_assert(ok); + + fs::wpath cachedRealPath; + ret = m_VFS->GetRealPath(L"cache"/cachedPath, cachedRealPath); + debug_assert(ret == INFO::OK); + + writer->AddFile(cachedRealPath, cachedPath.string()); + } + // TODO: should cache XML->XMB and DAE->PMD and DAE->PSA conversions too + else + { + debug_printf(L"Adding %ls\n", realPath.string().c_str()); + writer->AddFile(realPath, m_Files[i].string()); + } + } +} + +LibError CArchiveBuilder::CollectFileCB(const VfsPath& pathname, const FileInfo& UNUSED(fileInfo), const uintptr_t cbData) +{ + CArchiveBuilder* self = static_cast((void*)cbData); + self->m_Files.push_back(pathname); + + return INFO::OK; +} diff --git a/source/ps/ArchiveBuilder.h b/source/ps/ArchiveBuilder.h new file mode 100644 index 0000000000..5aed8341c2 --- /dev/null +++ b/source/ps/ArchiveBuilder.h @@ -0,0 +1,64 @@ +/* Copyright (C) 2010 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#ifndef INCLUDED_ARCHIVEBUILDER +#define INCLUDED_ARCHIVEBUILDER + +#include "lib/file/file_system_util.h" +#include "ps/CStr.h" + +/** + * Packages a mod's files into a distributable archive. + * This includes various game-specific knowledge on how to convert + * and cache certain files into more efficient formats (PNG -> DDS, etc). + */ +class CArchiveBuilder +{ +public: + /** + * Initialise the archive builder for processing the given mod. + * Assumes no graphics code (especially tex_codec) has been initialised yet. + * + * @param mod path to data/mods/foo directory, containing files for conversion + * @param tempdir path to a writable directory for temporary files + */ + CArchiveBuilder(const fs::wpath& mod, const fs::wpath& tempdir); + + ~CArchiveBuilder(); + + /** + * Add a mod which will be loaded but not archived, to provide + * files like textures.xml needed for the conversion. + * @param mod path to data/mods/foo directory, containing files for loading + */ + void AddBaseMod(const fs::wpath& mod); + + /** + * Do all the processing and packing of files into the archive. + * @param archive path of .zip file to generate (will be overwritten if it exists) + */ + void Build(const fs::wpath& archive); + +private: + static LibError CollectFileCB(const VfsPath& pathname, const FileInfo& fileInfo, const uintptr_t cbData); + + PIVFS m_VFS; + std::vector m_Files; + fs::wpath m_TempDir; +}; + +#endif // INCLUDED_ARCHIVEBUILDER diff --git a/source/ps/Filesystem.cpp b/source/ps/Filesystem.cpp index 99eec2e0d4..716687f986 100644 --- a/source/ps/Filesystem.cpp +++ b/source/ps/Filesystem.cpp @@ -29,9 +29,14 @@ PIVFS g_VFS; static std::vector > g_ReloadFuncs; +bool FileExists(const PIVFS& vfs, const VfsPath& pathname) +{ + return vfs->GetFileInfo(pathname, 0) == INFO::OK; +} + bool FileExists(const VfsPath& pathname) { - return g_VFS->GetFileInfo(pathname, 0) == INFO::OK; + return FileExists(g_VFS, pathname); } void RegisterFileReloadFunc(FileReloadFunc func, void* obj) diff --git a/source/ps/Filesystem.h b/source/ps/Filesystem.h index 319e985773..29a9f9b343 100644 --- a/source/ps/Filesystem.h +++ b/source/ps/Filesystem.h @@ -30,6 +30,8 @@ extern PIVFS g_VFS; +extern bool FileExists(const PIVFS& vfs, const VfsPath& pathname); + extern bool FileExists(const VfsPath& pathname); /** diff --git a/source/ps/XML/Xeromyces.cpp b/source/ps/XML/Xeromyces.cpp index fa26fc1333..54c29cf885 100644 --- a/source/ps/XML/Xeromyces.cpp +++ b/source/ps/XML/Xeromyces.cpp @@ -101,7 +101,7 @@ PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename) debug_assert(g_XeromycesStarted); // Make sure the .xml actually exists - if (! FileExists(filename)) + if (! FileExists(vfs, filename)) { LOG(CLogger::Error, LOG_CATEGORY, L"CXeromyces: Failed to find XML file %ls", filename.string().c_str()); return PSRETURN_Xeromyces_XMLOpenFailed; @@ -145,7 +145,7 @@ PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename) GetXMBPath(vfs, filename, xmbFilename, xmbPath); // If the file exists, use it - if (FileExists(xmbPath)) + if (FileExists(vfs, xmbPath)) { if (ReadXMBFile(vfs, xmbPath)) return PSRETURN_OK;