From 292975834b0bb2529efe3a3b291ed0bed39b0f5b Mon Sep 17 00:00:00 2001 From: janwas Date: Mon, 4 Feb 2008 11:40:42 +0000 Subject: [PATCH] # remove potential confusion that was the source of map load/store bug merge FilePacker and FileUnpacker into FileIo. add CppDoc documentation make clear what kind of 'size' is meant (file/buffer vs payload) This was SVN commit r5599. --- source/graphics/MapReader.h | 2 +- source/graphics/MapWriter.h | 2 +- source/graphics/MeshManager.cpp | 2 +- source/graphics/ModelDef.cpp | 3 +- source/graphics/SkeletonAnimDef.cpp | 3 +- source/graphics/SkeletonAnimManager.cpp | 2 +- source/ps/FileIo.cpp | 153 ++++++++++++++++++++++++ source/ps/FileIo.h | 123 +++++++++++++++++++ source/ps/FilePacker.cpp | 60 ---------- source/ps/FilePacker.h | 54 --------- source/ps/FileUnpacker.cpp | 107 ----------------- source/ps/FileUnpacker.h | 63 ---------- 12 files changed, 282 insertions(+), 292 deletions(-) create mode 100644 source/ps/FileIo.cpp create mode 100644 source/ps/FileIo.h delete mode 100644 source/ps/FilePacker.cpp delete mode 100644 source/ps/FilePacker.h delete mode 100644 source/ps/FileUnpacker.cpp delete mode 100644 source/ps/FileUnpacker.h diff --git a/source/graphics/MapReader.h b/source/graphics/MapReader.h index ba557dace6..15e01d115b 100644 --- a/source/graphics/MapReader.h +++ b/source/graphics/MapReader.h @@ -5,7 +5,7 @@ #include "lib/res/handle.h" #include "ps/CStr.h" #include "LightEnv.h" -#include "ps/FileUnpacker.h" +#include "ps/FileIo.h" class CObjectEntry; class CTerrain; diff --git a/source/graphics/MapWriter.h b/source/graphics/MapWriter.h index faff1d0166..878368f4aa 100644 --- a/source/graphics/MapWriter.h +++ b/source/graphics/MapWriter.h @@ -5,7 +5,7 @@ #include #include "MapIO.h" #include "ps/CStr.h" -#include "ps/FilePacker.h" +#include "ps/FileIo.h" class CLightEnv; diff --git a/source/graphics/MeshManager.cpp b/source/graphics/MeshManager.cpp index d380c6ad56..9cb24bedfc 100644 --- a/source/graphics/MeshManager.cpp +++ b/source/graphics/MeshManager.cpp @@ -5,7 +5,7 @@ #include "graphics/ColladaManager.h" #include "graphics/ModelDef.h" #include "ps/CLogger.h" -#include "ps/FileUnpacker.h" // to get access to its CError +#include "ps/FileIo.h" // to get access to its CError #include "ps/Profile.h" #define LOG_CATEGORY "mesh" diff --git a/source/graphics/ModelDef.cpp b/source/graphics/ModelDef.cpp index a5c1d85b64..e454ad5bec 100644 --- a/source/graphics/ModelDef.cpp +++ b/source/graphics/ModelDef.cpp @@ -10,8 +10,7 @@ #include "ModelDef.h" #include "graphics/SkeletonAnimDef.h" -#include "ps/FilePacker.h" -#include "ps/FileUnpacker.h" +#include "ps/FileIo.h" #include "maths/Vector4D.h" CVector3D CModelDef::SkinPoint(const SModelVertex& vtx, diff --git a/source/graphics/SkeletonAnimDef.cpp b/source/graphics/SkeletonAnimDef.cpp index 4fa0c5923a..638e4c794e 100644 --- a/source/graphics/SkeletonAnimDef.cpp +++ b/source/graphics/SkeletonAnimDef.cpp @@ -10,8 +10,7 @@ #include "SkeletonAnimDef.h" #include "maths/MathUtil.h" -#include "ps/FilePacker.h" -#include "ps/FileUnpacker.h" +#include "ps/FileIo.h" /////////////////////////////////////////////////////////////////////////////////////////// diff --git a/source/graphics/SkeletonAnimManager.cpp b/source/graphics/SkeletonAnimManager.cpp index 0e78d1c2d0..2ebcde6e90 100644 --- a/source/graphics/SkeletonAnimManager.cpp +++ b/source/graphics/SkeletonAnimManager.cpp @@ -14,7 +14,7 @@ #include "graphics/Model.h" #include "graphics/SkeletonAnimDef.h" #include "ps/CLogger.h" -#include "ps/FileUnpacker.h" +#include "ps/FileIo.h" #define LOG_CATEGORY "graphics" diff --git a/source/ps/FileIo.cpp b/source/ps/FileIo.cpp new file mode 100644 index 0000000000..f020c265cf --- /dev/null +++ b/source/ps/FileIo.cpp @@ -0,0 +1,153 @@ +/** + * ========================================================================= + * File : FileIo.cpp + * Project : 0 A.D. + * Description : endian-safe binary file IO helpers. + * ========================================================================= + */ + +#include "precompiled.h" + +#include "FileIo.h" +#include "ps/Filesystem.h" +#include "lib/byte_order.h" + +#pragma pack(push, 1) + +struct FileHeader +{ + char magic[4]; + u32 version_le; + u32 payloadSize_le; // = file size - sizeof(FileHeader) +}; +cassert(sizeof(FileHeader) == 12); + +#pragma pack(pop) + + +//----------------------------------------------------------------------------- +// CFilePacker + +CFilePacker::CFilePacker(u32 version, const char magic[4]) +{ + // put header in our data array. + // (its payloadSize_le will be updated on every Pack*() call) + FileHeader header; + strncpy(header.magic, magic, 4); // not 0-terminated => no _s + write_le32(&header.version_le, version); + write_le32(&header.payloadSize_le, 0); + m_writeBuffer.Append(&header, sizeof(FileHeader)); +} + + +CFilePacker::~CFilePacker() +{ +} + + +void CFilePacker::Write(const VfsPath& filename) +{ + const size_t payloadSize = m_writeBuffer.Size() - sizeof(FileHeader); + const u32 payloadSize_le = to_le32(u32_from_larger(payloadSize)); + m_writeBuffer.Overwrite(&payloadSize_le, sizeof(payloadSize_le), 0+offsetof(FileHeader, payloadSize_le)); + + // write out all data (including header) + if(g_VFS->CreateFile(filename, m_writeBuffer.Data(), m_writeBuffer.Size()) < 0) + throw PSERROR_File_WriteFailed(); +} + + +void CFilePacker::PackRaw(const void* rawData, size_t rawSize) +{ + m_writeBuffer.Append(rawData, rawSize); +} + + +void CFilePacker::PackString(const CStr& str) +{ + const size_t length = str.length(); + const u32 length_le = to_le32(u32_from_larger(length)); + PackRaw(&length_le, sizeof(length_le)); + PackRaw((const char*)str, length); +} + + +//----------------------------------------------------------------------------- +// CFileUnpacker + +CFileUnpacker::CFileUnpacker() + : m_bufSize(0), m_unpackPos(0), m_version(0) +{ +} + + +CFileUnpacker::~CFileUnpacker() +{ +} + + +void CFileUnpacker::Read(const VfsPath& filename, const char magic[4]) +{ + // if the file doesn't exist, LoadFile would raise an annoying error dialog. + // since this (unfortunately) happens so often, we perform a separate + // check and "just" raise an exception for the caller to handle. + // (this is nicer than somehow squelching internal VFS error reporting) + if(!FileExists(filename)) + throw PSERROR_File_OpenFailed(); + + // load the whole thing into memory + if(g_VFS->LoadFile(filename, m_buf, m_bufSize) < 0) + throw PSERROR_File_OpenFailed(); + + // make sure we read enough for the header + if(m_bufSize < sizeof(FileHeader)) + { + m_buf.reset(); + m_bufSize = 0; + throw PSERROR_File_ReadFailed(); + } + + // extract data from header + FileHeader* header = (FileHeader*)m_buf.get(); + m_version = read_le32(&header->version_le); + const size_t payloadSize = (size_t)read_le32(&header->payloadSize_le); + + // check we've got the right kind of file + // .. and that we read exactly headerSize+payloadSize + if(strncmp(header->magic, magic, 4) != 0 || m_bufSize != sizeof(FileHeader)+payloadSize) + { + m_buf.reset(); + m_bufSize = 0; + throw PSERROR_File_InvalidType(); + } + + m_unpackPos = sizeof(FileHeader); +} + + +void CFileUnpacker::UnpackRaw(void* rawData, size_t rawDataSize) +{ + // fail if reading past end of stream + if (m_unpackPos+rawDataSize > m_bufSize) + throw PSERROR_File_UnexpectedEOF(); + + void* src = m_buf.get() + m_unpackPos; + cpu_memcpy(rawData, src, rawDataSize); + m_unpackPos += rawDataSize; +} + + +void CFileUnpacker::UnpackString(CStr& result) +{ + // get string length + u32 length_le; + UnpackRaw(&length_le, sizeof(length_le)); + const size_t length = (size_t)to_le32(length_le); + + // fail if reading past end of stream + if (m_unpackPos+length > m_bufSize) + throw PSERROR_File_UnexpectedEOF(); + + result = CStr((char*)m_buf.get()+m_unpackPos, length); + m_unpackPos += length; +} diff --git a/source/ps/FileIo.h b/source/ps/FileIo.h new file mode 100644 index 0000000000..5a9efb3e0d --- /dev/null +++ b/source/ps/FileIo.h @@ -0,0 +1,123 @@ +/** + * ========================================================================= + * File : FileIo.h + * Project : 0 A.D. + * Description : endian-safe binary file IO helpers. + * ========================================================================= + */ + +// the file format has passing similarity to IFF. note however that +// "chunks" aren't identified by FOURCCs; only the file header is +// so marked. +// all > 8-bit integers are stored in little-endian format +// (hence the _le suffix). however, the caller is responsible for +// swapping their raw data before passing it to PackRaw. + +#ifndef INCLUDED_FILEPACKER +#define INCLUDED_FILEPACKER + +#include "CStr.h" +#include "lib/file/vfs/vfs_path.h" +#include "ps/Filesystem.h" // WriteBuffer + +#include "ps/Errors.h" +ERROR_GROUP(File); +ERROR_TYPE(File, OpenFailed); +ERROR_TYPE(File, WriteFailed); +ERROR_TYPE(File, InvalidType); +ERROR_TYPE(File, InvalidVersion); +ERROR_TYPE(File, ReadFailed); +ERROR_TYPE(File, UnexpectedEOF); + + +/** + * helper class for writing binary files. this is basically a + * resizable buffer that allows adding raw data and strings; + * upon calling Write(), everything is written out to disk. + **/ +class CFilePacker +{ +public: + /** + * adds version and signature (i.e. the header) to the buffer. + * this means Write() can write the entire buffer to file in one go, + * which is simpler and more efficient than writing in pieces. + **/ + CFilePacker(u32 version, const char magic[4]); + + ~CFilePacker(); + + /** + * write out to file all packed data added so far. + * it's safe to call this multiple times, but typically would + * only be done once. + **/ + void Write(const VfsPath& filename); + + /** + * pack given number of bytes onto the end of the data stream + **/ + void PackRaw(const void* rawData, size_t rawDataSize); + + /** + * pack a string onto the end of the data stream + * (encoded as a 32-bit length followed by the characters) + **/ + void PackString(const CStr& str); + +private: + /** + * the output data stream built during pack operations. + * contains the header, so we can write this out in one go. + **/ + WriteBuffer m_writeBuffer; +}; + + +/** + * helper class for reading binary files + **/ +class CFileUnpacker +{ +public: + CFileUnpacker(); + ~CFileUnpacker(); + + /** + * open and read in given file, check magic bits against those given; + * throw variety of exceptions if open failed / version incorrect, etc. + **/ + void Read(const VfsPath& filename, const char magic[4]); + + /** + * @return version number that was stored in the file's header. + **/ + u32 GetVersion() const + { + return m_version; + } + + /** + * unpack given number of bytes from the input into the given array. + * throws PSERROR_File_UnexpectedEOF if the end of the data stream is + * reached before the given number of bytes have been read. + **/ + void UnpackRaw(void* rawData, size_t rawDataSize); + + /** + * unpack a string from the raw data stream. + * @param result is assigned a newly constructed CStr8 holding the + * string read from the input stream. + **/ + void UnpackString(CStr8& result); + +private: + // the data read from file and used during unpack operations + shared_ptr m_buf; + size_t m_bufSize; + + size_t m_unpackPos; /// current unpack position in stream + u32 m_version; /// version that was stored in the file header +}; + +#endif diff --git a/source/ps/FilePacker.cpp b/source/ps/FilePacker.cpp deleted file mode 100644 index b8fdfd9de4..0000000000 --- a/source/ps/FilePacker.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/** - * ========================================================================= - * File : FilePacker.cpp - * Project : 0 A.D. - * Description : Resizable buffer, for writing binary files - * ========================================================================= - */ - -#include "precompiled.h" - -#include "FilePacker.h" -#include "ps/Filesystem.h" -#include "lib/byte_order.h" - -#include - - -//////////////////////////////////////////////////////////////////////////////////////// -// CFilePacker constructor -// rationale for passing in version + signature here: see header -CFilePacker::CFilePacker(u32 version, const char magicstr[4]) -{ - // put header in our data array. - // (size will be updated on every Pack*() call) - char header[12]; - strncpy(header+0, magicstr, 4); // not 0-terminated => no _s - write_le32(header+4, version); - write_le32(header+8, 0); // datasize - m_writeBuffer.Append(header, 12); -} - -//////////////////////////////////////////////////////////////////////////////////////// -// Write: write out to file all packed data added so far -void CFilePacker::Write(const VfsPath& filename) -{ - const u32 size_le = to_le32(u32_from_larger(m_writeBuffer.Size() - 12)); - m_writeBuffer.Overwrite(&size_le, sizeof(size_le), 8); - - // write out all data (including header) - if(g_VFS->CreateFile(filename, m_writeBuffer.Data(), m_writeBuffer.Size()) < 0) - throw PSERROR_File_WriteFailed(); -} - - -//////////////////////////////////////////////////////////////////////////////////////// -// PackRaw: pack given number of bytes onto the end of the data stream -void CFilePacker::PackRaw(const void* rawData, size_t rawSize) -{ - m_writeBuffer.Append(rawData, rawSize); -} - -//////////////////////////////////////////////////////////////////////////////////////// -// PackString: pack a string onto the end of the data stream -void CFilePacker::PackString(const CStr& str) -{ - const size_t length = str.length(); - const u32 length_le = to_le32(u32_from_larger(length)); - PackRaw(&length_le, sizeof(length_le)); - PackRaw((const char*)str, length); -} diff --git a/source/ps/FilePacker.h b/source/ps/FilePacker.h deleted file mode 100644 index da410bda91..0000000000 --- a/source/ps/FilePacker.h +++ /dev/null @@ -1,54 +0,0 @@ -/** - * ========================================================================= - * File : FilePacker.h - * Project : 0 A.D. - * Description : Resizable buffer, for writing binary files - * ========================================================================= - */ - -#ifndef INCLUDED_FILEPACKER -#define INCLUDED_FILEPACKER - -#include -#include "CStr.h" -#include "lib/file/vfs/vfs_path.h" - -#include "ps/Errors.h" -#include "ps/Filesystem.h" // WriteBuffer - -#ifndef ERROR_GROUP_FILE_DEFINED -#define ERROR_GROUP_FILE_DEFINED -// FileUnpacker.h defines these too -ERROR_GROUP(File); -ERROR_TYPE(File, OpenFailed); -#endif -ERROR_TYPE(File, WriteFailed); - -//////////////////////////////////////////////////////////////////////////////////////// -// CFilePacker: class to assist in writing of binary files. -// basically a resizeable buffer that allows adding raw data and strings; -// upon calling Write(), everything is written out to disk. -class CFilePacker -{ -public: - // constructor - // adds version and signature (i.e. the header) to the buffer. - // this means Write() can write the entire buffer to file in one go, - // which is simpler and more efficient than writing in pieces. - CFilePacker(u32 version, const char magicstr[4]); - - // Write: write out to file all packed data added so far - void Write(const VfsPath& filename); - - // PackRaw: pack given number of bytes onto the end of the data stream - void PackRaw(const void* rawdata, size_t rawdatalen); - // PackString: pack a string onto the end of the data stream - void PackString(const CStr& str); - -private: - // the output data stream built during pack operations. - // contains the header, so we can write this out in one go. - WriteBuffer m_writeBuffer; -}; - -#endif diff --git a/source/ps/FileUnpacker.cpp b/source/ps/FileUnpacker.cpp deleted file mode 100644 index c25e24a15d..0000000000 --- a/source/ps/FileUnpacker.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/** - * ========================================================================= - * File : FileUnpacker.cpp - * Project : 0 A.D. - * Description : Buffer and 'stream' for reading binary files - * ========================================================================= - */ - -#include "precompiled.h" - -#include "ps/FileUnpacker.h" -#include "ps/CStr.h" -#include "ps/Filesystem.h" -#include "lib/byte_order.h" - -//////////////////////////////////////////////////////////////////////////////////////// -// CFileUnpacker constructor -CFileUnpacker::CFileUnpacker() -{ - m_Size = 0; - m_UnpackPos = 0; - m_Version = 0; -} - -CFileUnpacker::~CFileUnpacker() -{ -} - -//////////////////////////////////////////////////////////////////////////////////////// -// Read: open and read in given file, check magic bits against those given; throw -// variety of exceptions for missing files etc -void CFileUnpacker::Read(const VfsPath& filename, const char magicstr[4]) -{ - // avoid vfs_load complaining about missing data files (which happens - // too often). better to check here than squelch internal VFS error - // reporting. we disable this in release mode to avoid a speed hit. - // UPDATE: We don't disable this in release mode, because vfs_load now - // complains about missing files when running in release -//#ifndef NDEBUG - if(!FileExists(filename)) - throw PSERROR_File_OpenFailed(); -//#endif - - // load the whole thing into memory - if(g_VFS->LoadFile(filename, m_Buf, m_Size) < 0) - throw PSERROR_File_OpenFailed(); - - // make sure we read enough for the header - if(m_Size < 12) - { - m_Buf.reset(); - m_Size = 0; - throw PSERROR_File_ReadFailed(); - } - - // extract data from header - u8* header = (u8*)m_Buf.get(); - char* magic = (char*)(header+0); - // FIXME m_Version and datasize: Byte order? -- Simon - m_Version = read_le32(header+4); - u32 datasize = read_le32(header+8); - - // check we've got the right kind of file - // .. and that we read exactly headersize+datasize - if(strncmp(magic, magicstr, 4) != 0 || m_Size != 12+datasize) - { - m_Buf.reset(); - m_Size = 0; - throw PSERROR_File_InvalidType(); - } - - m_UnpackPos = 12; -} - -//////////////////////////////////////////////////////////////////////////////////////// -// UnpackRaw: unpack given number of bytes from the input stream into the given array -// - throws CFileEOFError if the end of the data stream is reached before the given -// number of bytes have been read -void CFileUnpacker::UnpackRaw(void* rawdata, size_t rawdatalen) -{ - // fail if reading past end of stream - if (m_UnpackPos+rawdatalen > m_Size) - throw PSERROR_File_UnexpectedEOF(); - - void* src = m_Buf.get() + m_UnpackPos; - cpu_memcpy(rawdata, src, rawdatalen); - m_UnpackPos += rawdatalen; -} - -//////////////////////////////////////////////////////////////////////////////////////// -// UnpackString: unpack a string from the raw data stream -// - throws CFileEOFError if eof is reached before the string length has been -// satisfied -void CFileUnpacker::UnpackString(CStr& result) -{ - // get string length - u32 length_le; - UnpackRaw(&length_le, sizeof(length_le)); - const size_t length = to_le32(length_le); - - // fail if reading past end of stream - if (m_UnpackPos + length > m_Size) - throw PSERROR_File_UnexpectedEOF(); - - result = CStr((char*)m_Buf.get()+m_UnpackPos, length); - m_UnpackPos += length; -} diff --git a/source/ps/FileUnpacker.h b/source/ps/FileUnpacker.h deleted file mode 100644 index 3f96aa9cdf..0000000000 --- a/source/ps/FileUnpacker.h +++ /dev/null @@ -1,63 +0,0 @@ -/** - * ========================================================================= - * File : FileUnpacker.h - * Project : 0 A.D. - * Description : Buffer and 'stream' for reading binary files - * ========================================================================= - */ - -#ifndef INCLUDED_FILEUNPACKER -#define INCLUDED_FILEUNPACKER - -#include - -#include "lib/file/vfs/vfs_path.h" -class CStr8; - -#include "ps/Errors.h" -#ifndef ERROR_GROUP_FILE_DEFINED -#define ERROR_GROUP_FILE_DEFINED -// FilePacker.h defines these too -ERROR_GROUP(File); -ERROR_TYPE(File, OpenFailed); -#endif -ERROR_TYPE(File, InvalidType); -ERROR_TYPE(File, InvalidVersion); -ERROR_TYPE(File, ReadFailed); -ERROR_TYPE(File, UnexpectedEOF); - -//////////////////////////////////////////////////////////////////////////////// -// CFileUnpacker: class to assist in reading of binary files -class CFileUnpacker -{ -public: - // constructor - CFileUnpacker(); - - ~CFileUnpacker(); - - // Read: open and read in given file, check magic bits against those given; throw - // variety of exceptions for missing files etc - void Read(const VfsPath& filename, const char magicstr[4]); - - // GetVersion: return stored file version - u32 GetVersion() const { return m_Version; } - - // UnpackRaw: unpack given number of bytes from the input stream into the given array - // - throws PSERROR_File_UnexpectedEOF if the end of the data stream is reached before - // the given number of bytes have been read - void UnpackRaw(void* rawdata, size_t rawdatalen); - // UnpackString: unpack a string from the raw data stream - void UnpackString(CStr8& result); - -private: - // the data read from file and used during unpack operations - shared_ptr m_Buf; - size_t m_Size; - // current unpack position in stream - size_t m_UnpackPos; - // version of the file currently being read - u32 m_Version; -}; - -#endif