diff options
| -rw-r--r-- | src/SBaseCommon.cpp | 63 | ||||
| -rw-r--r-- | src/SBaseFileTable.cpp | 527 | ||||
| -rw-r--r-- | src/SBaseSubTypes.cpp | 2 | ||||
| -rw-r--r-- | src/SFileAddFile.cpp | 37 | ||||
| -rw-r--r-- | src/SFileAttributes.cpp | 17 | ||||
| -rw-r--r-- | src/SFileCompactArchive.cpp | 35 | ||||
| -rw-r--r-- | src/SFileFindFile.cpp | 98 | ||||
| -rw-r--r-- | src/SFileGetFileInfo.cpp | 7 | ||||
| -rw-r--r-- | src/SFileListFile.cpp | 15 | ||||
| -rw-r--r-- | src/SFileOpenArchive.cpp | 297 | ||||
| -rw-r--r-- | src/SFileReadFile.cpp | 2 | ||||
| -rw-r--r-- | src/SFileVerify.cpp | 17 | ||||
| -rw-r--r-- | src/StormCommon.h | 9 | ||||
| -rw-r--r-- | src/StormLib.h | 15 | ||||
| -rw-r--r-- | test/StormTest.cpp | 92 |
15 files changed, 614 insertions, 619 deletions
diff --git a/src/SBaseCommon.cpp b/src/SBaseCommon.cpp index a3b2a38..ed31a61 100644 --- a/src/SBaseCommon.cpp +++ b/src/SBaseCommon.cpp @@ -97,7 +97,7 @@ unsigned char AsciiToUpperTable_Slash[256] = #define STORM_BUFFER_SIZE 0x500 -#define HASH_INDEX_MASK(ha) (ha->pHeader->dwHashTableSize ? (ha->pHeader->dwHashTableSize - 1) : 0) +#define HASH_INDEX_MASK(ha) (ha->dwHashTableSize ? (ha->dwHashTableSize - 1) : 0) static DWORD StormBuffer[STORM_BUFFER_SIZE]; // Buffer for the decryption engine static bool bMpqCryptographyInitialized = false; @@ -712,9 +712,7 @@ TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry) if(ha != NULL && pFileEntry != NULL) { // Set the raw position and MPQ position - hf->RawFilePos = ha->MpqPos + pFileEntry->ByteOffset; - if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) - hf->RawFilePos = (DWORD)ha->MpqPos + (DWORD)pFileEntry->ByteOffset; + hf->RawFilePos = FileOffsetFromMpqOffset(ha, pFileEntry->ByteOffset); hf->MpqFilePos = pFileEntry->ByteOffset; // Set the data size @@ -733,8 +731,10 @@ void * LoadMpqTable( ULONGLONG ByteOffset, DWORD dwCompressedSize, DWORD dwTableSize, - DWORD dwKey) + DWORD dwKey, + bool * pbTableIsCut) { + ULONGLONG FileSize = 0; LPBYTE pbCompressed = NULL; LPBYTE pbMpqTable; LPBYTE pbToRead; @@ -757,19 +757,29 @@ void * LoadMpqTable( } } - // On archives v 1.0, Storm.dll does not actually check - // if the hash table was read entirely. Abused by Spazzler map protector - // which sets hash table size to 0x00100000 - if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1 && (ha->dwFlags & MPQ_FLAG_MALFORMED)) + // Get the file offset from which we will read the table + // Note: According to Storm.dll from Warcraft III (version 2002), + // if the hash table position is 0xFFFFFFFF, no SetFilePointer call is done + // and the table is loaded from the current file offset + if(ByteOffset == SFILE_INVALID_POS) + FileStream_GetPos(ha->pStream, &ByteOffset); + + // On archives v 1.0, hash table and block table can go beyond EOF. + // Storm.dll reads as much as possible, then fills the missing part with zeros. + // Abused by Spazzler map protector which sets hash table size to 0x00100000 + if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) { - ULONGLONG FileSize = 0; - // Cut the table size FileStream_GetSize(ha->pStream, &FileSize); if((ByteOffset + dwBytesToRead) > FileSize) { + // Fill the extra data with zeros dwBytesToRead = (DWORD)(FileSize - ByteOffset); memset(pbMpqTable + dwBytesToRead, 0, (dwTableSize - dwBytesToRead)); + + // Give the caller information that the table was cut + if(pbTableIsCut != NULL) + pbTableIsCut[0] = true; } } @@ -818,33 +828,6 @@ void * LoadMpqTable( return pbMpqTable; } -void CalculateRawSectorOffset( - ULONGLONG & RawFilePos, - TMPQFile * hf, - DWORD dwSectorOffset) -{ - // Must be used for files within a MPQ - assert(hf->ha != NULL); - assert(hf->ha->pHeader != NULL); - - // - // Some MPQ protectors place the sector offset table after the actual file data. - // Sector offsets in the sector offset table are negative. When added - // to MPQ file offset from the block table entry, the result is a correct - // position of the file data in the MPQ. - // - // For MPQs version 1.0, the offset is purely 32-bit - // - - RawFilePos = hf->RawFilePos + dwSectorOffset; - if(hf->ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) - RawFilePos = (DWORD)hf->ha->MpqPos + (DWORD)hf->pFileEntry->ByteOffset + dwSectorOffset; - - // We also have to add patch header size, if patch header is present - if(hf->pPatchInfo != NULL) - RawFilePos += hf->pPatchInfo->dwLength; -} - unsigned char * AllocateMd5Buffer( DWORD dwRawDataSize, DWORD dwChunkSize, @@ -1166,10 +1149,10 @@ int AllocateSectorChecksums(TMPQFile * hf, bool bLoadFromFile) // Calculate offset of the CRC table dwCrcSize = hf->dwSectorCount * sizeof(DWORD); dwCrcOffset = hf->SectorOffsets[hf->dwSectorCount]; - CalculateRawSectorOffset(RawFilePos, hf, dwCrcOffset); + RawFilePos = CalculateRawSectorOffset(hf, dwCrcOffset); // Now read the table from the MPQ - hf->SectorChksums = (DWORD *)LoadMpqTable(ha, RawFilePos, dwCompressedSize, dwCrcSize, 0); + hf->SectorChksums = (DWORD *)LoadMpqTable(ha, RawFilePos, dwCompressedSize, dwCrcSize, 0, NULL); if(hf->SectorChksums == NULL) return ERROR_NOT_ENOUGH_MEMORY; } diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp index 68ad0d7..635db08 100644 --- a/src/SBaseFileTable.cpp +++ b/src/SBaseFileTable.cpp @@ -286,6 +286,51 @@ static ULONGLONG DetermineArchiveSize_V2( return (EndOfMpq - MpqOffset); } +ULONGLONG FileOffsetFromMpqOffset(TMPQArchive * ha, ULONGLONG MpqOffset) +{ + if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) + { + // For MPQ archive v1, any file offset is only 32-bit + return (ULONGLONG)((DWORD)ha->MpqPos + (DWORD)MpqOffset); + } + else + { + // For MPQ archive v2+, file offsets are full 64-bit + return ha->MpqPos + MpqOffset; + } +} + +ULONGLONG CalculateRawSectorOffset( + TMPQFile * hf, + DWORD dwSectorOffset) +{ + ULONGLONG RawFilePos; + + // Must be used for files within a MPQ + assert(hf->ha != NULL); + assert(hf->ha->pHeader != NULL); + + // + // Some MPQ protectors place the sector offset table after the actual file data. + // Sector offsets in the sector offset table are negative. When added + // to MPQ file offset from the block table entry, the result is a correct + // position of the file data in the MPQ. + // + // For MPQs version 1.0, the offset is purely 32-bit + // + + RawFilePos = hf->RawFilePos + dwSectorOffset; + if(hf->ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) + RawFilePos = (DWORD)hf->ha->MpqPos + (DWORD)hf->pFileEntry->ByteOffset + dwSectorOffset; + + // We also have to add patch header size, if patch header is present + if(hf->pPatchInfo != NULL) + RawFilePos += hf->pPatchInfo->dwLength; + + // Return the result offset + return RawFilePos; +} + // This function converts the MPQ header so it always looks like version 4 int ConvertMpqHeaderToFormat4( TMPQArchive * ha, @@ -331,8 +376,6 @@ int ConvertMpqHeaderToFormat4( ha->dwFlags |= MPQ_FLAG_MALFORMED; if(pHeader->dwBlockTablePos <= pHeader->dwHeaderSize) ha->dwFlags |= MPQ_FLAG_MALFORMED; - if(pHeader->dwArchiveSize != pHeader->dwBlockTablePos + (pHeader->dwBlockTableSize * sizeof(TMPQBlock))) - ha->dwFlags |= MPQ_FLAG_MALFORMED; // Fill the rest of the header memset((LPBYTE)pHeader + MPQ_HEADER_SIZE_V1, 0, sizeof(TMPQHeader) - MPQ_HEADER_SIZE_V1); @@ -378,7 +421,7 @@ int ConvertMpqHeaderToFormat4( BlockTablePos64 = MAKE_OFFSET64(pHeader->wBlockTablePosHi, pHeader->dwBlockTablePos); // We require the block table to follow hash table - if(BlockTablePos64 > HashTablePos64) + if(BlockTablePos64 >= HashTablePos64) { // HashTableSize64 may be less than TblSize * sizeof(TMPQHash). // That means that the hash table is compressed. @@ -518,19 +561,7 @@ int ConvertMpqHeaderToFormat4( // Handle case when block table is placed before the MPQ header // Used by BOBA protector if(BlockTablePos64 < MpqOffset) - { - pHeader->BlockTableSize64 = (MpqOffset - BlockTablePos64) & BlockTableMask; - pHeader->dwBlockTableSize = (DWORD)(pHeader->BlockTableSize64 / sizeof(TMPQBlock)); - } - - // Handle case when either the MPQ is cut in the middle of the block table - // or the MPQ is malformed so that block table size is greater than should be - if((BlockTablePos64 + pHeader->BlockTableSize64) > FileSize) - { - pHeader->BlockTableSize64 = (FileSize - BlockTablePos64) & BlockTableMask; - pHeader->dwBlockTableSize = (DWORD)(pHeader->BlockTableSize64 / sizeof(TMPQBlock)); - } - + ha->dwFlags |= MPQ_FLAG_MALFORMED; return nError; } @@ -625,27 +656,30 @@ static int BuildFileTableFromBlockTable( TFileEntry * pFileEntry; TMPQHeader * pHeader = ha->pHeader; TMPQBlock * pBlock; - TMPQHash * pHashEnd = ha->pHashTable + pHeader->dwHashTableSize; + TMPQHash * pHashEnd = ha->pHashTable + ha->dwHashTableSize; TMPQHash * pHash; + // Parse the entire hash table for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) { + // + // Only if the block index is less than block table size. + // An MPQ protector can place multiple invalid entries + // into the hash table: + // + // a6d79af0 e61a0932 001e0000 0000770b <== Fake valid + // a6d79af0 e61a0932 0000d761 0000dacb <== Fake valid + // a6d79af0 e61a0932 00000000 0000002f <== Real file entry (block index is valid) + // a6d79af0 e61a0932 00005a4f 000093bc <== Fake valid + // if(pHash->dwBlockIndex < pHeader->dwBlockTableSize) { + // Get the pointer to the file entry and the block entry pFileEntry = pFileTable + pHash->dwBlockIndex; pBlock = pBlockTable + pHash->dwBlockIndex; - // - // Yet another silly map protector: For each valid file, - // there are 4 items in the hash table, that appears to be valid: - // - // a6d79af0 e61a0932 001e0000 0000770b <== Fake valid - // a6d79af0 e61a0932 0000d761 0000dacb <== Fake valid - // a6d79af0 e61a0932 00000000 0000002f <== Real file entry - // a6d79af0 e61a0932 00005a4f 000093bc <== Fake valid - // - - if(!(pBlock->dwFlags & ~MPQ_FILE_VALID_FLAGS) && (pBlock->dwFlags & MPQ_FILE_EXISTS)) + // Only if the block table entry contains valid file + if((pBlock->dwFlags & MPQ_FILE_EXISTS) && !(pBlock->dwFlags & ~MPQ_FILE_VALID_FLAGS)) { // ByteOffset is only valid if file size is not zero pFileEntry->ByteOffset = pBlock->dwFilePos; @@ -658,17 +692,16 @@ static int BuildFileTableFromBlockTable( pFileEntry->dwFlags = pBlock->dwFlags; pFileEntry->lcLocale = pHash->lcLocale; pFileEntry->wPlatform = pHash->wPlatform; + continue; } - else - { - // If the hash table entry doesn't point to the valid file item, - // we invalidate the hash table entry - pHash->dwName1 = 0xFFFFFFFF; - pHash->dwName2 = 0xFFFFFFFF; - pHash->lcLocale = 0xFFFF; - pHash->wPlatform = 0xFFFF; - pHash->dwBlockIndex = HASH_ENTRY_DELETED; - } + + // If the hash table entry doesn't point to the valid file item, + // we invalidate the hash table entry + pHash->dwName1 = 0xFFFFFFFF; + pHash->dwName2 = 0xFFFFFFFF; + pHash->lcLocale = 0xFFFF; + pHash->wPlatform = 0xFFFF; + pHash->dwBlockIndex = HASH_ENTRY_DELETED; } } @@ -680,7 +713,7 @@ static int UpdateFileTableFromHashTable( TFileEntry * pFileTable) { TFileEntry * pFileEntry; - TMPQHash * pHashEnd = ha->pHashTable + ha->pHeader->dwHashTableSize; + TMPQHash * pHashEnd = ha->pHashTable + ha->dwHashTableSize; TMPQHash * pHash; for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) @@ -708,11 +741,11 @@ static TMPQHash * TranslateHashTable( size_t HashTableSize; // Allocate copy of the hash table - pHashTable = STORM_ALLOC(TMPQHash, ha->pHeader->dwHashTableSize); + pHashTable = STORM_ALLOC(TMPQHash, ha->dwHashTableSize); if(pHashTable != NULL) { // Copy the hash table - HashTableSize = sizeof(TMPQHash) * ha->pHeader->dwHashTableSize; + HashTableSize = sizeof(TMPQHash) * ha->dwHashTableSize; memcpy(pHashTable, ha->pHashTable, HashTableSize); // Give the size to the caller @@ -1757,80 +1790,54 @@ void AllocateFileName(TMPQArchive * ha, TFileEntry * pFileEntry, const char * sz } } -TFileEntry * FindDeletedFileEntry(TMPQArchive * ha) -{ - TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; - TFileEntry * pFileEntry; - - // Go through the entire file table and try to find a deleted entry - for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++) - { - // Return the entry that is within the current file table (i.e. a deleted file) - if(!(pFileEntry->dwFlags & MPQ_FILE_EXISTS)) - return pFileEntry; - - // - // Note: Files with "delete marker" are not deleted. - // Don't treat them as available - // - } - - // No deleted entries found - return NULL; -} - TFileEntry * AllocateFileEntry(TMPQArchive * ha, const char * szFileName, LCID lcLocale) { + TFileEntry * pFileTableEnd; + TFileEntry * pFileTablePtr; + TFileEntry * pFileTable = ha->pFileTable; + TFileEntry * pLastEntry = ha->pFileTable; TFileEntry * pFileEntry = NULL; TMPQHash * pHash; + DWORD dwFreeNeeded = ha->dwReservedFiles; + DWORD dwFreeCount = 0; // The entry in the hash table should not be there. // This case must be handled by the caller assert(ha->pHashTable == NULL || GetHashEntryExact(ha, szFileName, lcLocale) == NULL); assert(ha->pHetTable == NULL || GetFileIndex_Het(ha, szFileName) == HASH_ENTRY_FREE); - // If we are in the middle of saving listfile, we need to allocate - // the file entry at the end of the file table - if((ha->dwFlags & MPQ_FLAG_SAVING_TABLES) == 0) + // Determine the end of the file table + pFileTableEnd = ha->pFileTable + STORMLIB_MAX(ha->dwFileTableSize, ha->dwMaxFileCount); + + // Find the first free file entry + for(pFileTablePtr = pFileTable; pFileTablePtr < pFileTableEnd; pFileTablePtr++) { - // Attempt to find a deleted file entry in the file table. - // If it suceeds, we reuse that entry - pFileEntry = FindDeletedFileEntry(ha); - if(pFileEntry == NULL) + // Is that entry free? + // Note: Files with "delete marker" are not deleted. + // Don't treat them as available + if((pFileTablePtr->dwFlags & MPQ_FILE_EXISTS) == 0) { - // If there is no space in the file table, we are sorry - if((ha->dwFileTableSize + ha->dwReservedFiles) >= ha->dwMaxFileCount) - return NULL; - - // Invalidate the internal files so we free - // their file entries. - InvalidateInternalFiles(ha); - - // Re-check for deleted entries - pFileEntry = FindDeletedFileEntry(ha); if(pFileEntry == NULL) - { - // If there is still no deleted entry, we allocate an entry at the end of the file table - assert((ha->dwFileTableSize + ha->dwReservedFiles) <= ha->dwMaxFileCount); - pFileEntry = ha->pFileTable + ha->dwFileTableSize++; - } + pFileEntry = pFileTablePtr; + dwFreeCount++; } else { - // Invalidate the internal files - InvalidateInternalFiles(ha); + // Update the last used entry + if(pFileTablePtr > pLastEntry) + pLastEntry = pFileTablePtr; } } - else - { - // There should be at least one entry for that internal file - assert((ha->dwFileTableSize + ha->dwReservedFiles) <= ha->dwMaxFileCount); - pFileEntry = ha->pFileTable + ha->dwFileTableSize++; - } // Did we find an usable file entry? if(pFileEntry != NULL) { + // Is there enough free entries? + if(!(ha->dwFlags & MPQ_FLAG_SAVING_TABLES)) + dwFreeNeeded++; + if(dwFreeCount < dwFreeNeeded) + return NULL; + // Make sure that the entry is properly initialized memset(pFileEntry, 0, sizeof(TFileEntry)); pFileEntry->lcLocale = (USHORT)lcLocale; @@ -1844,6 +1851,11 @@ TFileEntry * AllocateFileEntry(TMPQArchive * ha, const char * szFileName, LCID l pHash = AllocateHashEntry(ha, pFileEntry); assert(pHash != NULL); } + + // Update the file table size + if(pFileEntry >= pLastEntry) + pLastEntry = pFileEntry; + ha->dwFileTableSize = (DWORD)(pLastEntry - pFileTable) + 1; } // Return the file entry @@ -1860,7 +1872,7 @@ int RenameFileEntry( // Mark the entry as deleted in the hash table if(ha->pHashTable != NULL) { - assert(pFileEntry->dwHashIndex < ha->pHeader->dwHashTableSize); + assert(pFileEntry->dwHashIndex < ha->dwHashTableSize); pHash = ha->pHashTable + pFileEntry->dwHashIndex; memset(pHash, 0xFF, sizeof(TMPQHash)); @@ -1909,7 +1921,7 @@ void DeleteFileEntry( { // We expect dwHashIndex to be within the hash table pHash = ha->pHashTable + pFileEntry->dwHashIndex; - assert(pFileEntry->dwHashIndex < ha->pHeader->dwHashTableSize); + assert(pFileEntry->dwHashIndex < ha->dwHashTableSize); // Set the hash table entry as deleted pHash->dwName1 = 0xFFFFFFFF; @@ -1940,7 +1952,6 @@ void DeleteFileEntry( void InvalidateInternalFiles(TMPQArchive * ha) { - TFileEntry * pFileTableEnd; TFileEntry * pFileEntry1 = NULL; TFileEntry * pFileEntry2 = NULL; TFileEntry * pFileEntry3 = NULL; @@ -1955,62 +1966,39 @@ void InvalidateInternalFiles(TMPQArchive * ha) // // Invalidate the (listfile), if not done yet - if(ha->dwFileFlags1 != 0 && (ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) == 0) + if((ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) == 0) { - // Delete the existing entry for (listfile) + // If the (listfile) is not there but will be, reserve one entry pFileEntry1 = GetFileEntryExact(ha, LISTFILE_NAME, LANG_NEUTRAL); - if(pFileEntry1 != NULL) - DeleteFileEntry(ha, pFileEntry1); - - // Reserve one entry for (listfile) + if(pFileEntry1 == NULL && ha->dwFileFlags1) + ha->dwReservedFiles++; + + // Mark the (listfile) as invalid ha->dwFlags |= MPQ_FLAG_LISTFILE_INVALID; - ha->dwReservedFiles++; } // Invalidate the (attributes), if not done yet - if(ha->dwFileFlags2 != 0 && (ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) == 0) + if((ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) == 0) { - // Delete the existing entry for (attributes) + // If the (attributes) is not there but will be, reserve one entry pFileEntry2 = GetFileEntryExact(ha, ATTRIBUTES_NAME, LANG_NEUTRAL); - if(pFileEntry2 != NULL) - DeleteFileEntry(ha, pFileEntry2); + if(pFileEntry2 == NULL && ha->dwFileFlags2) + ha->dwReservedFiles++; - // Reserve one entry for (attributes) + // Mark (attributes) as invalid ha->dwFlags |= MPQ_FLAG_ATTRIBUTES_INVALID; - ha->dwReservedFiles++; } // Invalidate the (signature), if not done yet - if(ha->dwFileFlags3 != 0 && (ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) == 0) + if((ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) == 0) { - // Delete the existing entry for (attributes) + // If the (signature) is not there but will be, reserve one entry pFileEntry3 = GetFileEntryExact(ha, SIGNATURE_NAME, LANG_NEUTRAL); - if(pFileEntry3 != NULL) - DeleteFileEntry(ha, pFileEntry3); + if(pFileEntry3 == NULL && ha->dwFileFlags3) + ha->dwReservedFiles++; - // Reserve one entry for (attributes) + // Mark (signature) as invalid ha->dwFlags |= MPQ_FLAG_SIGNATURE_INVALID; - ha->dwReservedFiles++; - } - - - // If the internal files are at the end of the file table (they usually are), - // we want to free these 3 entries, so when new files are added, they get - // added to the freed entries and the internal files get added after that - if(ha->dwFileTableSize > 0) - { - pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; - - // Is one of the entries the last one? - if(pFileEntry1 == pFileTableEnd - 1 || pFileEntry2 == pFileTableEnd - 1 || pFileEntry3 == pFileTableEnd - 1) - pFileTableEnd--; - if(pFileEntry1 == pFileTableEnd - 1 || pFileEntry2 == pFileTableEnd - 1 || pFileEntry3 == pFileTableEnd - 1) - pFileTableEnd--; - if(pFileEntry1 == pFileTableEnd - 1 || pFileEntry2 == pFileTableEnd - 1 || pFileEntry3 == pFileTableEnd - 1) - pFileTableEnd--; - - // Calculate the new file table size - ha->dwFileTableSize = (DWORD)(pFileTableEnd - ha->pFileTable); } // Remember that the MPQ has been changed @@ -2021,93 +2009,6 @@ void InvalidateInternalFiles(TMPQArchive * ha) //----------------------------------------------------------------------------- // Support for file tables - hash table, block table, hi-block table -// Fixes the case when hash table size is set to arbitrary long value -static void FixHashTableSize(TMPQArchive * ha, ULONGLONG ByteOffset) -{ - ULONGLONG FileSize = 0; - DWORD dwFixedTableSize; - DWORD dwHashTableSize = ha->pHeader->dwHashTableSize; - - // Spazzler protector abuses the fact that hash table size does not matter - // if the hash table is entirely filled with values different than 0xFFFFFFFF. - // It sets the hash table size to 0x00100000, which slows down file searching - // and adding a listfile. - - // Only if the hash table size is correct - if((dwHashTableSize & (dwHashTableSize - 1)) == 0) - { - // Retrieve the file size - FileStream_GetSize(ha->pStream, &FileSize); - - // Work as long as the size greater than file size - for(;;) - { - // Try the size of one half of the current - dwFixedTableSize = dwHashTableSize >> 1; - if(ByteOffset + (dwFixedTableSize * sizeof(TMPQHash)) <= FileSize) - break; - - // Cut the hash table size to half - dwHashTableSize = dwFixedTableSize; - } - - // Fix the hash table size - ha->pHeader->dwHashTableSize = dwHashTableSize; - ha->pHeader->HashTableSize64 = dwHashTableSize * sizeof(TMPQHash); - } -} - -// This function fixes the scenario then dwBlockTableSize -// is greater and goes into a MPQ file -static void FixBlockTableSize( - TMPQArchive * ha, - TMPQBlock * pBlockTable) -{ - TMPQHeader * pHeader = ha->pHeader; - ULONGLONG BlockTableStart; - ULONGLONG BlockTableEnd; - ULONGLONG FileDataStart; - DWORD dwFixedTableSize = pHeader->dwBlockTableSize; - - // Only perform this check on MPQs version 1.0 - assert(pHeader->dwHeaderSize == MPQ_HEADER_SIZE_V1); - - // Calculate claimed block table begin and end - BlockTableStart = ha->MpqPos + MAKE_OFFSET64(pHeader->wBlockTablePosHi, pHeader->dwBlockTablePos); - BlockTableEnd = BlockTableStart + (pHeader->dwBlockTableSize * sizeof(TMPQBlock)); - - for(DWORD i = 0; i < dwFixedTableSize; i++) - { - // If the block table end goes into that file, fix the block table end - FileDataStart = ha->MpqPos + pBlockTable[i].dwFilePos; - if(BlockTableStart < FileDataStart && BlockTableEnd > FileDataStart) - { - dwFixedTableSize = (DWORD)((FileDataStart - BlockTableStart) / sizeof(TMPQBlock)); - BlockTableEnd = FileDataStart; - ha->dwFlags |= MPQ_FLAG_MALFORMED; - } - } - - // If we found a mismatch in the block table size - if(dwFixedTableSize < pHeader->dwBlockTableSize) - { - // Fill the additional space with zeros - memset(pBlockTable + dwFixedTableSize, 0, (pHeader->dwBlockTableSize - dwFixedTableSize) * sizeof(TMPQBlock)); - - // Fix the block table size - pHeader->dwBlockTableSize = dwFixedTableSize; - pHeader->BlockTableSize64 = dwFixedTableSize * sizeof(TMPQBlock); - - // - // Note: We should recalculate the archive size in the header, - // (it might be invalid as well) but we don't care about it anyway - // - - // In theory, a MPQ could have bigger block table than hash table - assert(pHeader->dwBlockTableSize <= ha->dwMaxFileCount); - } -} - int CreateHashTable(TMPQArchive * ha, DWORD dwHashTableSize) { TMPQHash * pHashTable; @@ -2127,6 +2028,7 @@ int CreateHashTable(TMPQArchive * ha, DWORD dwHashTableSize) // Fill it memset(pHashTable, 0xFF, dwHashTableSize * sizeof(TMPQHash)); + ha->dwHashTableSize = dwHashTableSize; ha->dwMaxFileCount = dwHashTableSize; ha->pHashTable = pHashTable; return ERROR_SUCCESS; @@ -2139,6 +2041,7 @@ TMPQHash * LoadHashTable(TMPQArchive * ha) TMPQHash * pHashTable = NULL; DWORD dwTableSize; DWORD dwCmpSize; + bool bHashTableIsCut = false; // If the MPQ has no hash table, do nothing if(pHeader->dwHashTablePos == 0 && pHeader->wHashTablePosHi == 0) @@ -2153,29 +2056,17 @@ TMPQHash * LoadHashTable(TMPQArchive * ha) { case MPQ_SUBTYPE_MPQ: - ByteOffset = ha->MpqPos + MAKE_OFFSET64(pHeader->wHashTablePosHi, pHeader->dwHashTablePos); - if((pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) && (ha->dwFlags & MPQ_FLAG_MALFORMED)) - { - // Calculate the hash table offset as 32-bit value - ByteOffset = (DWORD)ha->MpqPos + pHeader->dwHashTablePos; - - // Defense against map protectors that set the hash table size too big - FixHashTableSize(ha, ByteOffset); - } - - // Get the compressed and uncompressed hash table size + // Calculate the position and size of the hash table + ByteOffset = FileOffsetFromMpqOffset(ha, MAKE_OFFSET64(pHeader->wHashTablePosHi, pHeader->dwHashTablePos)); dwTableSize = pHeader->dwHashTableSize * sizeof(TMPQHash); dwCmpSize = (DWORD)pHeader->HashTableSize64; - // - // Load the table from the MPQ, with decompression - // - // Note: We will NOT check if the hash table is properly decrypted. - // Some MPQ protectors corrupt the hash table by rewriting part of it. - // Hash table, the way how it works, allows arbitrary values for unused entries. - // + // Read, decrypt and uncompress the hash table + pHashTable = (TMPQHash *)LoadMpqTable(ha, ByteOffset, dwCmpSize, dwTableSize, MPQ_KEY_HASH_TABLE, &bHashTableIsCut); - pHashTable = (TMPQHash *)LoadMpqTable(ha, ByteOffset, dwCmpSize, dwTableSize, MPQ_KEY_HASH_TABLE); + // If the hash table was cut, we need to remember it + if(pHashTable != NULL && bHashTableIsCut) + ha->dwFlags |= (MPQ_FLAG_MALFORMED | MPQ_FLAG_HASH_TABLE_CUT); break; case MPQ_SUBTYPE_SQP: @@ -2187,7 +2078,7 @@ TMPQHash * LoadHashTable(TMPQArchive * ha) break; } - // Return the hash table + // Remember the size of the hash table return pHashTable; } @@ -2198,6 +2089,7 @@ TMPQBlock * LoadBlockTable(TMPQArchive * ha, bool /* bDontFixEntries */) ULONGLONG ByteOffset; DWORD dwTableSize; DWORD dwCmpSize; + bool bBlockTableIsCut = false; // Do nothing if the block table position is zero if(pHeader->dwBlockTablePos == 0 && pHeader->wBlockTablePosHi == 0) @@ -2213,37 +2105,16 @@ TMPQBlock * LoadBlockTable(TMPQArchive * ha, bool /* bDontFixEntries */) case MPQ_SUBTYPE_MPQ: // Calculate byte position of the block table - ByteOffset = ha->MpqPos + MAKE_OFFSET64(pHeader->wBlockTablePosHi, pHeader->dwBlockTablePos); - if((pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) && (ha->dwFlags & MPQ_FLAG_MALFORMED)) - ByteOffset = (DWORD)ha->MpqPos + pHeader->dwBlockTablePos; - - // Calculate full and compressed size of the block table + ByteOffset = FileOffsetFromMpqOffset(ha, MAKE_OFFSET64(pHeader->wBlockTablePosHi, pHeader->dwBlockTablePos)); dwTableSize = pHeader->dwBlockTableSize * sizeof(TMPQBlock); dwCmpSize = (DWORD)pHeader->BlockTableSize64; - assert((ByteOffset + dwCmpSize) <= (ha->MpqPos + ha->pHeader->ArchiveSize64)); - // - // One of the first cracked versions of Diablo I had block table unencrypted - // StormLib does NOT support such MPQs anymore, as they are incompatible - // with compressed block table feature - // - - // If failed to load the block table, delete it - pBlockTable = (TMPQBlock * )LoadMpqTable(ha, ByteOffset, dwCmpSize, dwTableSize, MPQ_KEY_BLOCK_TABLE); - - // De-protecting maps for Warcraft III - if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) - { - // Some protectors set the block table size bigger than really is - if(pBlockTable != NULL) - FixBlockTableSize(ha, pBlockTable); - - // - // Note: TMPQBlock::dwCSize file size can be completely invalid - // for compressed files and the file can still be read. - // Abused by some MPQ protectors - // - } + // Read, decrypt and uncompress the block table + pBlockTable = (TMPQBlock * )LoadMpqTable(ha, ByteOffset, dwCmpSize, dwTableSize, MPQ_KEY_BLOCK_TABLE, &bBlockTableIsCut); + + // If the block table was cut, we need to remember it + if(pBlockTable != NULL && bBlockTableIsCut) + ha->dwFlags |= (MPQ_FLAG_MALFORMED | MPQ_FLAG_BLOCK_TABLE_CUT); break; case MPQ_SUBTYPE_SQP: @@ -2329,10 +2200,11 @@ int LoadAnyHashTable(TMPQArchive * ha) // Note: We don't care about HET table limits, because HET table is rebuilt // after each file add/rename/delete. ha->dwMaxFileCount = (ha->pHashTable != NULL) ? pHeader->dwHashTableSize : HASH_TABLE_SIZE_MAX; + ha->dwHashTableSize = pHeader->dwHashTableSize; return ERROR_SUCCESS; } -int BuildFileTable_Classic( +static int BuildFileTable_Classic( TMPQArchive * ha, TFileEntry * pFileTable) { @@ -2416,7 +2288,7 @@ int BuildFileTable_Classic( return nError; } -int BuildFileTable_HetBet( +static int BuildFileTable_HetBet( TMPQArchive * ha, TFileEntry * pFileTable) { @@ -2580,6 +2452,113 @@ int BuildFileTable(TMPQArchive * ha) return ERROR_SUCCESS; } +// Defragment the file table so it does not contain any gaps +int DefragmentFileTable(TMPQArchive * ha) +{ + TFileEntry * pFileTable = ha->pFileTable; + TFileEntry * pSrcEntry = pFileTable; + TFileEntry * pTrgEntry = pFileTable; + + // Parse the file entries + for(DWORD i = 0; i < ha->dwFileTableSize; i++, pSrcEntry++) + { + // Is the source file entry does not contain anything valid, we skip it + if(pSrcEntry->dwFlags & MPQ_FILE_EXISTS) + { + // Fix the block table index and move the entry + if(pTrgEntry != pSrcEntry) + { + if(ha->pHashTable != NULL) + ha->pHashTable[pSrcEntry->dwHashIndex].dwBlockIndex = (DWORD)(pTrgEntry - pFileTable); + memmove(pTrgEntry, pSrcEntry, sizeof(TFileEntry)); + } + + // Move the target pointer by one entry + pTrgEntry++; + } + } + + // Store the new size of the file table + ha->dwFileTableSize = (DWORD)(pTrgEntry - pFileTable); + + // Fill the rest with zeros + if(ha->dwFileTableSize < ha->dwMaxFileCount) + memset(pFileTable + ha->dwFileTableSize, 0, (ha->dwMaxFileCount - ha->dwFileTableSize) * sizeof(TFileEntry)); + return ERROR_SUCCESS; +} + +// Performs final validation (and possible defragmentation) +// of malformed hash+block tables +int ShrinkMalformedMpqTables(TMPQArchive * ha) +{ + TFileEntry * pFileTable = ha->pFileTable; + TMPQHash * pHashTable = ha->pHashTable; + DWORD i; + + // Sanity checks + assert(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1); + assert(ha->pHashTable != NULL); + assert(ha->pFileTable != NULL); + + // + // Note: Unhandled possible case where multiple hash table entries + // point to the same block tabl entry. + // + + // Now perform the defragmentation of the block table + if(ha->dwFlags & MPQ_FLAG_BLOCK_TABLE_CUT) + { + assert(ha->dwFileTableSize == ha->pHeader->dwBlockTableSize); + DefragmentFileTable(ha); + } + + // If the hash table has been cut, we need to defragment it + // This will greatly improve performance on searching and loading listfile + if(ha->dwFlags & MPQ_FLAG_HASH_TABLE_CUT) + { + TMPQHash * pSrcEntry = ha->pHashTable; + TMPQHash * pTrgEntry = ha->pHashTable; + DWORD dwNewEntryCount; + DWORD dwNewTableSize; + + assert(ha->dwHashTableSize == ha->pHeader->dwHashTableSize); + + // Defragment the hash table + for(i = 0; i < ha->dwHashTableSize; i++, pSrcEntry++) + { + // If the entry is valid, move it + if(pSrcEntry->dwBlockIndex < ha->dwFileTableSize) + { + // Fix the hash table index in the file table and move the entry + if(pTrgEntry != pSrcEntry) + { + pFileTable[pSrcEntry->dwBlockIndex].dwHashIndex = (DWORD)(pTrgEntry - pHashTable); + *pTrgEntry = *pSrcEntry; + } + + // Increment the number of used entries + pTrgEntry++; + } + } + + // Calculate the new hash table size + dwNewEntryCount = (DWORD)(pTrgEntry - pHashTable); + dwNewTableSize = GetHashTableSizeForFileCount(dwNewEntryCount); + + // Mark the extra entries in the hash table as deleted + // This causes every hash search to go over the entire hash table, + // but if the hash table is cut, it would do it anyway + memset(pHashTable + dwNewEntryCount, 0xFF, (dwNewTableSize - dwNewEntryCount) * sizeof(TMPQHash)); + for(i = dwNewEntryCount; i < dwNewTableSize; i++) + pHashTable[i].dwBlockIndex = HASH_ENTRY_DELETED; + + // Change the hash table size in the header + ha->dwHashTableSize = dwNewTableSize; + } + + return ERROR_SUCCESS; +} + // Rebuilds the HET table from scratch based on the file table // Used after a modifying operation (add, rename, delete) int RebuildHetTable(TMPQArchive * ha) @@ -2628,7 +2607,7 @@ int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize, DWORD dwNewMaxF int nError = ERROR_SUCCESS; // The new hash table size must be greater or equal to the current hash table size - assert(dwNewHashTableSize >= ha->pHeader->dwHashTableSize); + assert(dwNewHashTableSize >= ha->dwHashTableSize); // The new hash table size must be a power of two assert((dwNewHashTableSize & (dwNewHashTableSize - 1)) == 0); @@ -2658,7 +2637,7 @@ int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize, DWORD dwNewMaxF ha->pHashTable = pHashTable; // Set the new limits to the MPQ archive - ha->pHeader->dwHashTableSize = dwNewHashTableSize; + ha->dwHashTableSize = dwNewHashTableSize; ha->dwMaxFileCount = dwNewMaxFileCount; // Now copy all the file entries diff --git a/src/SBaseSubTypes.cpp b/src/SBaseSubTypes.cpp index 117221d..9807701 100644 --- a/src/SBaseSubTypes.cpp +++ b/src/SBaseSubTypes.cpp @@ -525,7 +525,7 @@ TMPQHash * LoadMpkHashTable(TMPQArchive * ha) if(pMpkHash != NULL) { // Calculate the hash table size as if it was real MPQ hash table - pHeader->dwHashTableSize = GetHashTableSizeForFileCount(dwHashTableSize); + pHeader->dwHashTableSize = GetHashTableSizeForFileCount(pHeader->dwHashTableSize); pHeader->HashTableSize64 = pHeader->dwHashTableSize * sizeof(TMPQHash); // Now allocate table that will serve like a true MPQ hash table, diff --git a/src/SFileAddFile.cpp b/src/SFileAddFile.cpp index 0878ab7..aa47aa7 100644 --- a/src/SFileAddFile.cpp +++ b/src/SFileAddFile.cpp @@ -321,7 +321,7 @@ static int RecryptFileData( } // Calculate the raw file offset of the file sector - CalculateRawSectorOffset(RawFilePos, hf, dwRawByteOffset); + RawFilePos = CalculateRawSectorOffset(hf, dwRawByteOffset); // Read the file sector if(!FileStream_Read(ha->pStream, &RawFilePos, hf->pbFileSector, dwRawDataInSector)) @@ -410,9 +410,8 @@ int SFileAddFile_Init( if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) { TempPos = hf->MpqFilePos + dwFileSize; - TempPos += ha->pHeader->dwHashTableSize * sizeof(TMPQHash); - TempPos += ha->pHeader->dwBlockTableSize * sizeof(TMPQBlock); - TempPos += ha->pHeader->dwBlockTableSize * sizeof(USHORT); + TempPos += ha->dwHashTableSize * sizeof(TMPQHash); + TempPos += ha->dwFileTableSize * sizeof(TMPQBlock); if((TempPos >> 32) != 0) nError = ERROR_DISK_FULL; } @@ -425,6 +424,10 @@ int SFileAddFile_Init( pFileEntry = GetFileEntryExact(ha, szFileName, lcLocale); if(pFileEntry == NULL) { + // First, free the internal files + InvalidateInternalFiles(ha); + + // Find a free entry in the file table pFileEntry = AllocateFileEntry(ha, szFileName, lcLocale); if(pFileEntry == NULL) nError = ERROR_DISK_FULL; @@ -432,13 +435,10 @@ int SFileAddFile_Init( else { // If the caller didn't set MPQ_FILE_REPLACEEXISTING, fail it - if((dwFlags & MPQ_FILE_REPLACEEXISTING) == 0) - nError = ERROR_ALREADY_EXISTS; - - // When replacing an existing file, - // we still need to invalidate the (attributes) file - if(nError == ERROR_SUCCESS) + if(dwFlags & MPQ_FILE_REPLACEEXISTING) InvalidateInternalFiles(ha); + else + nError = ERROR_ALREADY_EXISTS; } } @@ -725,23 +725,6 @@ bool WINAPI SFileCreateFile( nError = ERROR_INVALID_PARAMETER; } - // The number of files must not overflow the maximum - // Example: size of block table: 0x41, size of hash table: 0x40 - if(nError == ERROR_SUCCESS) - { - DWORD dwReservedFiles = ha->dwReservedFiles; - - if(dwReservedFiles == 0) - { - dwReservedFiles += ha->dwFileFlags1 ? 1 : 0; - dwReservedFiles += ha->dwFileFlags2 ? 1 : 0; - dwReservedFiles += ha->dwFileFlags3 ? 1 : 0; - } - - if((ha->dwFileTableSize + dwReservedFiles) > ha->dwMaxFileCount) - nError = ERROR_DISK_FULL; - } - // Initiate the add file operation if(nError == ERROR_SUCCESS) nError = SFileAddFile_Init(ha, szArchivedName, FileTime, dwFileSize, lcLocale, dwFlags, (TMPQFile **)phFile); diff --git a/src/SFileAttributes.cpp b/src/SFileAttributes.cpp index 0c5dd06..81bf1cf 100644 --- a/src/SFileAttributes.cpp +++ b/src/SFileAttributes.cpp @@ -264,7 +264,7 @@ static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile) } // Allocate the buffer for holding the entire (attributes) - // Allodate 1 byte more (See GetSizeOfAttributesFile for more info) + // Allocate 1 byte more (See GetSizeOfAttributesFile for more info) cbAttrFile = GetSizeOfAttributesFile(ha->dwAttrFlags, dwFinalEntries); pbAttrFile = pbAttrPtr = STORM_ALLOC(BYTE, cbAttrFile + 1); if(pbAttrFile != NULL) @@ -349,7 +349,7 @@ static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile) // Now we expect that current position matches the estimated size // Note that if there is 1 extra bit above the byte size, - // the table is actually 1 byte shorted in Blizzard MPQs. See GetSizeOfAttributesFile + // the table is actually 1 byte shorter in Blizzard MPQs. See GetSizeOfAttributesFile assert((size_t)(pbAttrPtr - pbAttrFile) == cbAttrFile); } @@ -420,7 +420,7 @@ int SAttrFileSaveToMpq(TMPQArchive * ha) { // At this point, we expect to have at least one reserved entry in the file table assert(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID); - assert(ha->dwReservedFiles >= 1); + assert(ha->dwReservedFiles > 0); // Create the raw data that is to be written to (attributes) // Note: Blizzard MPQs have entries for (listfile) and (attributes), @@ -456,16 +456,13 @@ int SAttrFileSaveToMpq(TMPQArchive * ha) } else { - // If the list file is empty, we assume ERROR_SUCCESS + // If the (attributes) file would be empty, its OK nError = (cbAttrFile == 0) ? ERROR_SUCCESS : ERROR_NOT_ENOUGH_MEMORY; } - // If the save process succeeded, we clear the MPQ_FLAG_ATTRIBUTE_INVALID flag - if(nError == ERROR_SUCCESS) - { - ha->dwFlags &= ~MPQ_FLAG_ATTRIBUTES_INVALID; - ha->dwReservedFiles--; - } + // Clear the number of reserved files + ha->dwFlags &= ~MPQ_FLAG_ATTRIBUTES_INVALID; + ha->dwReservedFiles--; } return nError; diff --git a/src/SFileCompactArchive.cpp b/src/SFileCompactArchive.cpp index 238873c..a70fa0a 100644 --- a/src/SFileCompactArchive.cpp +++ b/src/SFileCompactArchive.cpp @@ -20,7 +20,7 @@ static int CheckIfAllFilesKnown(TMPQArchive * ha) { - TFileEntry * pFileTableEnd; + TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; TFileEntry * pFileEntry; DWORD dwBlockIndex = 0; int nError = ERROR_SUCCESS; @@ -28,7 +28,6 @@ static int CheckIfAllFilesKnown(TMPQArchive * ha) // Verify the file table if(nError == ERROR_SUCCESS) { - pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++, dwBlockIndex++) { // If there is an existing entry in the file table, check its name @@ -49,7 +48,7 @@ static int CheckIfAllFilesKnown(TMPQArchive * ha) static int CheckIfAllKeysKnown(TMPQArchive * ha, const char * szListFile, LPDWORD pFileKeys) { - TFileEntry * pFileTableEnd; + TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; TFileEntry * pFileEntry; DWORD dwBlockIndex = 0; int nError = ERROR_SUCCESS; @@ -67,7 +66,6 @@ static int CheckIfAllKeysKnown(TMPQArchive * ha, const char * szListFile, LPDWOR // Verify the file table if(nError == ERROR_SUCCESS) { - pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++, dwBlockIndex++) { // If the file exists and it's encrypted @@ -266,7 +264,7 @@ static int CopyMpqFileSectors( dwRawDataInSector = dwBytesToCopy; // Calculate the raw file offset of the file sector - CalculateRawSectorOffset(RawFilePos, hf, dwRawByteOffset); + RawFilePos = CalculateRawSectorOffset(hf, dwRawByteOffset); // Read the file sector if(!FileStream_Read(ha->pStream, &RawFilePos, hf->pbFileSector, dwRawDataInSector)) @@ -600,19 +598,13 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR if(nError == ERROR_SUCCESS) nError = CopyMpqFiles(ha, pFileKeys, pTempStream); - // Defragment the file table - if(nError == ERROR_SUCCESS) - nError = RebuildFileTable(ha, ha->pHeader->dwHashTableSize, ha->dwMaxFileCount); - // We also need to rebuild the HET table, if any if(nError == ERROR_SUCCESS) { - // Invalidate (listfile) and (attributes) - InvalidateInternalFiles(ha); - // Rebuild the HET table, if we have any if(ha->pHetTable != NULL) nError = RebuildHetTable(ha); + ha->dwFlags |= MPQ_FLAG_CHANGED; } // If succeeded, switch the streams @@ -624,21 +616,12 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR nError = ERROR_CAN_NOT_COMPLETE; } - // If all succeeded, save the MPQ tables - if(nError == ERROR_SUCCESS) + // Final user notification + if(nError == ERROR_SUCCESS && ha->pfnCompactCB != NULL) { - // - // Note: We don't recalculate position of the MPQ tables at this point. - // SaveMPQTables does it automatically. - // - - nError = SaveMPQTables(ha); - if(nError == ERROR_SUCCESS && ha->pfnCompactCB != NULL) - { - ha->CompactBytesProcessed += (ha->pHeader->dwHashTableSize * sizeof(TMPQHash)); - ha->CompactBytesProcessed += (ha->pHeader->dwBlockTableSize * sizeof(TMPQBlock)); - ha->pfnCompactCB(ha->pvCompactUserData, CCB_CLOSING_ARCHIVE, ha->CompactBytesProcessed, ha->CompactTotalBytes); - } + ha->CompactBytesProcessed += (ha->dwHashTableSize * sizeof(TMPQHash)); + ha->CompactBytesProcessed += (ha->dwFileTableSize * sizeof(TMPQBlock)); + ha->pfnCompactCB(ha->pvCompactUserData, CCB_CLOSING_ARCHIVE, ha->CompactBytesProcessed, ha->CompactTotalBytes); } // Cleanup and return diff --git a/src/SFileFindFile.cpp b/src/SFileFindFile.cpp index 80a9e59..0f7bd39 100644 --- a/src/SFileFindFile.cpp +++ b/src/SFileFindFile.cpp @@ -213,14 +213,6 @@ static bool FileWasFoundBefore( return false; } -static inline bool FileEntryIsInvalid( - TMPQArchive * ha, - TFileEntry * pFileEntry) -{ - // Spazzler3 protector: Some files are clearly wrong - return ((ha->dwFlags & MPQ_FLAG_MALFORMED) && (pFileEntry->dwCmpSize & 0xFFFF0000) >= 0x7FFF0000); -} - static TFileEntry * FindPatchEntry(TMPQArchive * ha, TFileEntry * pFileEntry) { TFileEntry * pPatchEntry = NULL; @@ -282,60 +274,56 @@ static int DoMPQSearch(TMPQSearch * hs, SFILE_FIND_DATA * lpFindFileData) // Is it a file but not a patch file? if((pFileEntry->dwFlags & hs->dwFlagMask) == MPQ_FILE_EXISTS) { - // Spazzler3 protector: Some files are clearly wrong - if(!FileEntryIsInvalid(ha, pFileEntry)) - { - // Now we have to check if this file was not enumerated before - if(!FileWasFoundBefore(ha, hs, pFileEntry)) - { -// if(pFileEntry != NULL && !_stricmp(pFileEntry->szFileName, "TriggerLibs\\NativeLib.galaxy")) -// DebugBreak(); + // Now we have to check if this file was not enumerated before + if(!FileWasFoundBefore(ha, hs, pFileEntry)) + { +// if(pFileEntry != NULL && !_stricmp(pFileEntry->szFileName, "TriggerLibs\\NativeLib.galaxy")) +// DebugBreak(); - // Find a patch to this file - pPatchEntry = FindPatchEntry(ha, pFileEntry); - if(pPatchEntry == NULL) - pPatchEntry = pFileEntry; + // Find a patch to this file + pPatchEntry = FindPatchEntry(ha, pFileEntry); + if(pPatchEntry == NULL) + pPatchEntry = pFileEntry; - // Prepare the block index - dwBlockIndex = (DWORD)(pFileEntry - ha->pFileTable); + // Prepare the block index + dwBlockIndex = (DWORD)(pFileEntry - ha->pFileTable); - // Get the file name. If it's not known, we will create pseudo-name - szFileName = pFileEntry->szFileName; - if(szFileName == NULL) + // Get the file name. If it's not known, we will create pseudo-name + szFileName = pFileEntry->szFileName; + if(szFileName == NULL) + { + // Open the file by its pseudo-name. + // This also generates the file name with a proper extension + sprintf(szPseudoName, "File%08u.xxx", (unsigned int)dwBlockIndex); + if(SFileOpenFileEx((HANDLE)hs->ha, szPseudoName, SFILE_OPEN_BASE_FILE, &hFile)) { - // Open the file by its pseudo-name. - // This also generates the file name with a proper extension - sprintf(szPseudoName, "File%08u.xxx", (unsigned int)dwBlockIndex); - if(SFileOpenFileEx((HANDLE)hs->ha, szPseudoName, SFILE_OPEN_BASE_FILE, &hFile)) - { - szFileName = (pFileEntry->szFileName != NULL) ? pFileEntry->szFileName : szPseudoName; - SFileCloseFile(hFile); - } + szFileName = (pFileEntry->szFileName != NULL) ? pFileEntry->szFileName : szPseudoName; + SFileCloseFile(hFile); } + } - // If the file name is still NULL, we cannot include the file to search results - if(szFileName != NULL) + // If the file name is still NULL, we cannot include the file to search results + if(szFileName != NULL) + { + // Check the file name against the wildcard + if(CheckWildCard(szFileName + nPrefixLength, hs->szSearchMask)) { - // Check the file name against the wildcard - if(CheckWildCard(szFileName + nPrefixLength, hs->szSearchMask)) - { - // Fill the found entry. hash entry and block index are taken from the base MPQ - lpFindFileData->dwHashIndex = pFileEntry->dwHashIndex; - lpFindFileData->dwBlockIndex = dwBlockIndex; - lpFindFileData->dwFileSize = pPatchEntry->dwFileSize; - lpFindFileData->dwFileFlags = pPatchEntry->dwFlags; - lpFindFileData->dwCompSize = pPatchEntry->dwCmpSize; - lpFindFileData->lcLocale = pPatchEntry->lcLocale; - - // Fill the filetime - lpFindFileData->dwFileTimeHi = (DWORD)(pPatchEntry->FileTime >> 32); - lpFindFileData->dwFileTimeLo = (DWORD)(pPatchEntry->FileTime); - - // Fill the file name and plain file name - strcpy(lpFindFileData->cFileName, szFileName + nPrefixLength); - lpFindFileData->szPlainName = (char *)GetPlainFileName(lpFindFileData->cFileName); - return ERROR_SUCCESS; - } + // Fill the found entry. hash entry and block index are taken from the base MPQ + lpFindFileData->dwHashIndex = pFileEntry->dwHashIndex; + lpFindFileData->dwBlockIndex = dwBlockIndex; + lpFindFileData->dwFileSize = pPatchEntry->dwFileSize; + lpFindFileData->dwFileFlags = pPatchEntry->dwFlags; + lpFindFileData->dwCompSize = pPatchEntry->dwCmpSize; + lpFindFileData->lcLocale = pPatchEntry->lcLocale; + + // Fill the filetime + lpFindFileData->dwFileTimeHi = (DWORD)(pPatchEntry->FileTime >> 32); + lpFindFileData->dwFileTimeLo = (DWORD)(pPatchEntry->FileTime); + + // Fill the file name and plain file name + strcpy(lpFindFileData->cFileName, szFileName + nPrefixLength); + lpFindFileData->szPlainName = (char *)GetPlainFileName(lpFindFileData->cFileName); + return ERROR_SUCCESS; } } } diff --git a/src/SFileGetFileInfo.cpp b/src/SFileGetFileInfo.cpp index e93aff1..9cf8038 100644 --- a/src/SFileGetFileInfo.cpp +++ b/src/SFileGetFileInfo.cpp @@ -376,11 +376,12 @@ bool WINAPI SFileGetFileInfo( if(ha != NULL) { nInfoType = SFILE_INFO_TYPE_NOT_FOUND; - if(ha->pHashTable != NULL) + if(MAKE_OFFSET64(ha->pHeader->wHashTablePosHi, ha->pHeader->dwHashTablePos) != 0) { - pvSrcFileInfo = ha->pHashTable; cbSrcFileInfo = ha->pHeader->dwHashTableSize * sizeof(TMPQHash); - nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER; + if(cbFileInfo >= cbSrcFileInfo) + pvSrcFileInfo = LoadHashTable(ha); + nInfoType = SFILE_INFO_TYPE_ALLOCATED; } } break; diff --git a/src/SFileListFile.cpp b/src/SFileListFile.cpp index f4069c0..ab6dff2 100644 --- a/src/SFileListFile.cpp +++ b/src/SFileListFile.cpp @@ -354,6 +354,8 @@ static int SListFileCreateNodeForAllLocales(TMPQArchive * ha, const char * szFil // Now find the next language version of the file pHash = GetNextHashEntry(ha, pFirstHash, pHash); } + + return ERROR_SUCCESS; } return ERROR_CAN_NOT_COMPLETE; @@ -372,7 +374,7 @@ int SListFileSaveToMpq(TMPQArchive * ha) { // At this point, we expect to have at least one reserved entry in the file table assert(ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID); - assert(ha->dwReservedFiles >= 1); + assert(ha->dwReservedFiles > 0); // Create the raw data that is to be written to (listfile) // Note: Creating the raw data before the (listfile) has been created in the MPQ @@ -409,16 +411,13 @@ int SListFileSaveToMpq(TMPQArchive * ha) } else { - // If the list file is empty, we assume ERROR_SUCCESS + // If the (listfile) file would be empty, its OK nError = (cbListFile == 0) ? ERROR_SUCCESS : ERROR_NOT_ENOUGH_MEMORY; } - // If the save process succeeded, we clear the MPQ_FLAG_LISTFILE_INVALID flag - if(nError == ERROR_SUCCESS) - { - ha->dwFlags &= ~MPQ_FLAG_LISTFILE_INVALID; - ha->dwReservedFiles--; - } + // Clear the listfile flags + ha->dwFlags &= ~MPQ_FLAG_LISTFILE_INVALID; + ha->dwReservedFiles--; } return nError; diff --git a/src/SFileOpenArchive.cpp b/src/SFileOpenArchive.cpp index dd178e5..35e12d6 100644 --- a/src/SFileOpenArchive.cpp +++ b/src/SFileOpenArchive.cpp @@ -17,6 +17,8 @@ #include "StormLib.h" #include "StormCommon.h" +#define HEADER_SEARCH_BUFFER_SIZE 0x1000 + /*****************************************************************************/ /* Local functions */ /*****************************************************************************/ @@ -91,10 +93,7 @@ static int VerifyMpqTablePositions(TMPQArchive * ha, ULONGLONG FileSize) // Check the begin of hash table if(pHeader->wHashTablePosHi || pHeader->dwHashTablePos) { - ByteOffset = ha->MpqPos + MAKE_OFFSET64(pHeader->wHashTablePosHi, pHeader->dwHashTablePos); - if((pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) && (ha->dwFlags & MPQ_FLAG_MALFORMED)) - ByteOffset = (DWORD)ha->MpqPos + pHeader->dwHashTablePos; - + ByteOffset = FileOffsetFromMpqOffset(ha, MAKE_OFFSET64(pHeader->wHashTablePosHi, pHeader->dwHashTablePos)); if(ByteOffset > FileSize) return ERROR_BAD_FORMAT; } @@ -102,10 +101,7 @@ static int VerifyMpqTablePositions(TMPQArchive * ha, ULONGLONG FileSize) // Check the begin of block table if(pHeader->wBlockTablePosHi || pHeader->dwBlockTablePos) { - ByteOffset = ha->MpqPos + MAKE_OFFSET64(pHeader->wBlockTablePosHi, pHeader->dwBlockTablePos); - if((pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) && (ha->dwFlags & MPQ_FLAG_MALFORMED)) - ByteOffset = (DWORD)ha->MpqPos + pHeader->dwBlockTablePos; - + ByteOffset = FileOffsetFromMpqOffset(ha, MAKE_OFFSET64(pHeader->wBlockTablePosHi, pHeader->dwBlockTablePos)); if(ByteOffset > FileSize) return ERROR_BAD_FORMAT; } @@ -161,6 +157,7 @@ bool WINAPI SFileOpenArchive( TMPQArchive * ha = NULL; // Archive handle TFileEntry * pFileEntry; ULONGLONG FileSize = 0; // Size of the file + LPBYTE pbHeaderBuffer = NULL; // Buffer for searching MPQ header bool bIsWarcraft3Map = false; int nError = ERROR_SUCCESS; @@ -201,7 +198,15 @@ bool WINAPI SFileOpenArchive( nError = ERROR_NOT_ENOUGH_MEMORY; } - // Initialize handle structure and allocate structure for MPQ header + // Allocate buffer for searching MPQ header + if(nError == ERROR_SUCCESS) + { + pbHeaderBuffer = STORM_ALLOC(BYTE, HEADER_SEARCH_BUFFER_SIZE); + if(pbHeaderBuffer == NULL) + nError = ERROR_NOT_ENOUGH_MEMORY; + } + + // Find the position of MPQ header if(nError == ERROR_SUCCESS) { ULONGLONG SearchOffset = 0; @@ -209,6 +214,7 @@ bool WINAPI SFileOpenArchive( DWORD dwStreamFlags = 0; DWORD dwHeaderSize; DWORD dwHeaderID; + bool bSearchComplete = false; memset(ha, 0, sizeof(TMPQArchive)); ha->pfnHashString = HashString; @@ -230,16 +236,19 @@ bool WINAPI SFileOpenArchive( EndOfSearch = 0x08000000; // Find the offset of MPQ header within the file - while(SearchOffset < EndOfSearch) + while(bSearchComplete == false && SearchOffset < EndOfSearch) { - DWORD dwBytesAvailable = MPQ_HEADER_SIZE_V4; + // Always read at least 0x1000 bytes for performance. + // This is what Storm.dll (2002) does. + DWORD dwBytesAvailable = HEADER_SEARCH_BUFFER_SIZE; + DWORD dwInBufferOffset = 0; // Cut the bytes available, if needed - if((FileSize - SearchOffset) < MPQ_HEADER_SIZE_V4) + if((FileSize - SearchOffset) < HEADER_SEARCH_BUFFER_SIZE) dwBytesAvailable = (DWORD)(FileSize - SearchOffset); // Read the eventual MPQ header - if(!FileStream_Read(ha->pStream, &SearchOffset, ha->HeaderData, dwBytesAvailable)) + if(!FileStream_Read(ha->pStream, &SearchOffset, pbHeaderBuffer, dwBytesAvailable)) { nError = GetLastError(); break; @@ -248,67 +257,78 @@ bool WINAPI SFileOpenArchive( // There are AVI files from Warcraft III with 'MPQ' extension. if(SearchOffset == 0) { - if(IsAviFile(ha->HeaderData)) + if(IsAviFile((DWORD *)pbHeaderBuffer)) { nError = ERROR_AVI_FILE; break; } - bIsWarcraft3Map = IsWarcraft3Map(ha->HeaderData); + bIsWarcraft3Map = IsWarcraft3Map((DWORD *)pbHeaderBuffer); } - // If there is the MPQ user data, process it - // Note that Warcraft III does not check for user data, which is abused by many map protectors - dwHeaderID = BSWAP_INT32_UNSIGNED(ha->HeaderData[0]); - if(bIsWarcraft3Map == false && (dwFlags & MPQ_OPEN_FORCE_MPQ_V1) == 0) + // Search the header buffer + while(dwInBufferOffset < dwBytesAvailable) { - if(ha->pUserData == NULL && dwHeaderID == ID_MPQ_USERDATA) + // Copy the data from the potential header buffer to the MPQ header + memcpy(ha->HeaderData, pbHeaderBuffer + dwInBufferOffset, sizeof(ha->HeaderData)); + + // If there is the MPQ user data, process it + // Note that Warcraft III does not check for user data, which is abused by many map protectors + dwHeaderID = BSWAP_INT32_UNSIGNED(ha->HeaderData[0]); + if(bIsWarcraft3Map == false && (dwFlags & MPQ_OPEN_FORCE_MPQ_V1) == 0) { - // Verify if this looks like a valid user data - pUserData = IsValidMpqUserData(SearchOffset, FileSize, ha->HeaderData); - if(pUserData != NULL) + if(ha->pUserData == NULL && dwHeaderID == ID_MPQ_USERDATA) { - // Fill the user data header - ha->UserDataPos = SearchOffset; - ha->pUserData = &ha->UserData; - memcpy(ha->pUserData, pUserData, sizeof(TMPQUserData)); - - // Continue searching from that position - SearchOffset += ha->pUserData->dwHeaderOffs; - continue; + // Verify if this looks like a valid user data + pUserData = IsValidMpqUserData(SearchOffset, FileSize, ha->HeaderData); + if(pUserData != NULL) + { + // Fill the user data header + ha->UserDataPos = SearchOffset; + ha->pUserData = &ha->UserData; + memcpy(ha->pUserData, pUserData, sizeof(TMPQUserData)); + + // Continue searching from that position + SearchOffset += ha->pUserData->dwHeaderOffs; + break; + } } } - } - // There must be MPQ header signature. Note that STORM.dll from Warcraft III actually - // tests the MPQ header size. It must be at least 0x20 bytes in order to load it - // Abused by Spazzler Map protector. Note that the size check is not present - // in Storm.dll v 1.00, so Diablo I code would load the MPQ anyway. - dwHeaderSize = BSWAP_INT32_UNSIGNED(ha->HeaderData[1]); - if(dwHeaderID == ID_MPQ && dwHeaderSize >= MPQ_HEADER_SIZE_V1) - { - // Now convert the header to version 4 - nError = ConvertMpqHeaderToFormat4(ha, SearchOffset, FileSize, dwFlags); - break; - } + // There must be MPQ header signature. Note that STORM.dll from Warcraft III actually + // tests the MPQ header size. It must be at least 0x20 bytes in order to load it + // Abused by Spazzler Map protector. Note that the size check is not present + // in Storm.dll v 1.00, so Diablo I code would load the MPQ anyway. + dwHeaderSize = BSWAP_INT32_UNSIGNED(ha->HeaderData[1]); + if(dwHeaderID == ID_MPQ && dwHeaderSize >= MPQ_HEADER_SIZE_V1) + { + // Now convert the header to version 4 + nError = ConvertMpqHeaderToFormat4(ha, SearchOffset, FileSize, dwFlags); + bSearchComplete = true; + break; + } - // Check for MPK archives (Longwu Online - MPQ fork) - if(dwHeaderID == ID_MPK) - { - // Now convert the MPK header to MPQ Header version 4 - nError = ConvertMpkHeaderToFormat4(ha, FileSize, dwFlags); - break; - } + // Check for MPK archives (Longwu Online - MPQ fork) + if(dwHeaderID == ID_MPK) + { + // Now convert the MPK header to MPQ Header version 4 + nError = ConvertMpkHeaderToFormat4(ha, FileSize, dwFlags); + bSearchComplete = true; + break; + } - // If searching for the MPQ header is disabled, return an error - if(dwFlags & MPQ_OPEN_NO_HEADER_SEARCH) - { - nError = ERROR_NOT_SUPPORTED; - break; - } + // If searching for the MPQ header is disabled, return an error + if(dwFlags & MPQ_OPEN_NO_HEADER_SEARCH) + { + nError = ERROR_NOT_SUPPORTED; + bSearchComplete = true; + break; + } - // Move to the next possible offset - SearchOffset += 0x200; + // Move the pointers + SearchOffset += 0x200; + dwInBufferOffset += 0x200; + } } // Did we identify one of the supported headers? @@ -371,38 +391,12 @@ bool WINAPI SFileOpenArchive( nError = BuildFileTable(ha); } - // Verify the file table, if no kind of malformation was detected - if(nError == ERROR_SUCCESS && (ha->dwFlags & MPQ_FLAG_MALFORMED) == 0) + // We now have both hash table and block table loaded. + // If any malformation was detected in the MPQ, + // we perform hash table and block table validation (and eventual defragmentation) + if(nError == ERROR_SUCCESS && ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1 && (ha->dwFlags & MPQ_FLAG_MALFORMED)) { - TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; - ULONGLONG RawFilePos; - - // Parse all file entries - for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++) - { - // If that file entry is valid, check the file position - if(pFileEntry->dwFlags & MPQ_FILE_EXISTS) - { - // Get the 64-bit file position, - // relative to the begin of the file - RawFilePos = ha->MpqPos + pFileEntry->ByteOffset; - - // Begin of the file must be within range - if(RawFilePos > FileSize) - { - nError = ERROR_FILE_CORRUPT; - break; - } - - // End of the file must be within range - RawFilePos += pFileEntry->dwCmpSize; - if(RawFilePos > FileSize) - { - nError = ERROR_FILE_CORRUPT; - break; - } - } - } + nError = ShrinkMalformedMpqTables(ha); } // Load the internal listfile and include it to the file table @@ -442,6 +436,9 @@ bool WINAPI SFileOpenArchive( assert(pFileEntry->dwFlags == MPQ_FILE_EXISTS); ha->dwFileFlags3 = pFileEntry->dwFlags; } + + // Finally, set the MPQ_FLAG_READ_ONLY if the MPQ was found malformed + ha->dwFlags |= (ha->dwFlags & MPQ_FLAG_MALFORMED) ? MPQ_FLAG_READ_ONLY : 0; } // Cleanup and exit @@ -453,6 +450,9 @@ bool WINAPI SFileOpenArchive( ha = NULL; } + // Free the header buffer + if(pbHeaderBuffer != NULL) + STORM_FREE(pbHeaderBuffer); *phMpq = ha; return (nError == ERROR_SUCCESS); } @@ -488,6 +488,7 @@ bool WINAPI SFileSetDownloadCallback(HANDLE hMpq, SFILE_DOWNLOAD_CALLBACK Downlo bool WINAPI SFileFlushArchive(HANDLE hMpq) { + TFileEntry * pFileEntry; TMPQArchive * ha; int nResultError = ERROR_SUCCESS; int nError; @@ -499,52 +500,98 @@ bool WINAPI SFileFlushArchive(HANDLE hMpq) return false; } - // Indicate that we are saving MPQ internal structures - ha->dwFlags |= MPQ_FLAG_SAVING_TABLES; - - // If the (signature) has been invalidated, save it - if(ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) + // Only if the MPQ was changed + if(ha->dwFlags & MPQ_FLAG_CHANGED) { - nError = SSignFileCreate(ha); - if(nError != ERROR_SUCCESS) - nResultError = nError; - } + // Indicate that we are saving MPQ internal structures + ha->dwFlags |= MPQ_FLAG_SAVING_TABLES; - // If the (listfile) has been invalidated, save it - if(ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) - { - nError = SListFileSaveToMpq(ha); - if(nError != ERROR_SUCCESS) - nResultError = nError; - } + // + // Invalidate entries for each internal file + // - // If the (attributes) has been invalidated, save it - if(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) - { - nError = SAttrFileSaveToMpq(ha); - if(nError != ERROR_SUCCESS) - nResultError = nError; - } + if(ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) + { + pFileEntry = GetFileEntryExact(ha, LISTFILE_NAME, LANG_NEUTRAL); + if(pFileEntry != NULL) + { + DeleteFileEntry(ha, pFileEntry); + ha->dwReservedFiles++; + } + } - // Save HET table, BET table, hash table, block table, hi-block table - if(ha->dwFlags & MPQ_FLAG_CHANGED) - { - // Save all MPQ tables first - nError = SaveMPQTables(ha); - if(nError != ERROR_SUCCESS) - nResultError = nError; + if(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) + { + pFileEntry = GetFileEntryExact(ha, ATTRIBUTES_NAME, LANG_NEUTRAL); + if(pFileEntry != NULL) + { + DeleteFileEntry(ha, pFileEntry); + ha->dwReservedFiles++; + } + } + + if(ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) + { + pFileEntry = GetFileEntryExact(ha, SIGNATURE_NAME, LANG_NEUTRAL); + if(pFileEntry != NULL) + { + DeleteFileEntry(ha, pFileEntry); + ha->dwReservedFiles++; + } + } + + // + // Defragment the file table. This will allow us to put the internal files to the end + // - // If the archive has weak signature, we need to finish it - if(ha->dwFileFlags3 != 0) + DefragmentFileTable(ha); + + // + // Create each internal file + // Note that the (signature) file is usually before (listfile) in the file table + // + + if(ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) { - nError = SSignFileFinish(ha); + nError = SSignFileCreate(ha); if(nError != ERROR_SUCCESS) nResultError = nError; } - } - // We are no longer saving internal MPQ structures - ha->dwFlags &= ~MPQ_FLAG_SAVING_TABLES; + if(ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) + { + nError = SListFileSaveToMpq(ha); + if(nError != ERROR_SUCCESS) + nResultError = nError; + } + + if(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) + { + nError = SAttrFileSaveToMpq(ha); + if(nError != ERROR_SUCCESS) + nResultError = nError; + } + + // Save HET table, BET table, hash table, block table, hi-block table + if(ha->dwFlags & MPQ_FLAG_CHANGED) + { + // Save all MPQ tables first + nError = SaveMPQTables(ha); + if(nError != ERROR_SUCCESS) + nResultError = nError; + + // If the archive has weak signature, we need to finish it + if(ha->dwFileFlags3 != 0) + { + nError = SSignFileFinish(ha); + if(nError != ERROR_SUCCESS) + nResultError = nError; + } + } + + // We are no longer saving internal MPQ structures + ha->dwFlags &= ~MPQ_FLAG_SAVING_TABLES; + } // Return the error if(nResultError != ERROR_SUCCESS) diff --git a/src/SFileReadFile.cpp b/src/SFileReadFile.cpp index 75c5508..02def18 100644 --- a/src/SFileReadFile.cpp +++ b/src/SFileReadFile.cpp @@ -92,7 +92,7 @@ static int ReadMpqSectors(TMPQFile * hf, LPBYTE pbBuffer, DWORD dwByteOffset, DW } // Calculate raw file offset where the sector(s) are stored. - CalculateRawSectorOffset(RawFilePos, hf, dwRawSectorOffset); + RawFilePos = CalculateRawSectorOffset(hf, dwRawSectorOffset); // Set file pointer and read all required sectors if(FileStream_Read(ha->pStream, &RawFilePos, pbInSector, dwRawBytesToRead)) diff --git a/src/SFileVerify.cpp b/src/SFileVerify.cpp index 8cf138a..cff2a81 100644 --- a/src/SFileVerify.cpp +++ b/src/SFileVerify.cpp @@ -787,6 +787,7 @@ int SSignFileCreate(TMPQArchive * ha) { // The (signature) file must be non-encrypted and non-compressed assert(ha->dwFileFlags3 == MPQ_FILE_EXISTS); + assert(ha->dwReservedFiles > 0); // Create the (signature) file file in the MPQ // Note that the file must not be compressed or encrypted @@ -800,21 +801,15 @@ int SSignFileCreate(TMPQArchive * ha) // Write the empty signature file to the archive if(nError == ERROR_SUCCESS) { - // Write the empty zeroed fiel to the MPQ + // Write the empty zeroed file to the MPQ memset(EmptySignature, 0, sizeof(EmptySignature)); nError = SFileAddFile_Write(hf, EmptySignature, (DWORD)sizeof(EmptySignature), 0); + SFileAddFile_Finish(hf); } - // If the save process succeeded, we clear the MPQ_FLAG_ATTRIBUTE_INVALID flag - if(nError == ERROR_SUCCESS) - { - ha->dwFlags &= ~MPQ_FLAG_SIGNATURE_INVALID; - ha->dwReservedFiles--; - } - - // Free the file - if(hf != NULL) - SFileAddFile_Finish(hf); + // Clear the invalid mark + ha->dwFlags &= ~MPQ_FLAG_SIGNATURE_INVALID; + ha->dwReservedFiles--; } return nError; diff --git a/src/StormCommon.h b/src/StormCommon.h index dbd1d0d..cdca0b0 100644 --- a/src/StormCommon.h +++ b/src/StormCommon.h @@ -170,6 +170,9 @@ TMPQFile * IsValidFileHandle(HANDLE hFile); //----------------------------------------------------------------------------- // Support for MPQ file tables +ULONGLONG FileOffsetFromMpqOffset(TMPQArchive * ha, ULONGLONG MpqOffset); +ULONGLONG CalculateRawSectorOffset(TMPQFile * hf, DWORD dwSectorOffset); + int ConvertMpqHeaderToFormat4(TMPQArchive * ha, ULONGLONG MpqOffset, ULONGLONG FileSize, DWORD dwFlags); TMPQHash * FindFreeHashEntry(TMPQArchive * ha, DWORD dwStartIndex, DWORD dwName1, DWORD dwName2, LCID lcLocale); @@ -191,6 +194,9 @@ ULONGLONG FindFreeMpqSpace(TMPQArchive * ha); int CreateHashTable(TMPQArchive * ha, DWORD dwHashTableSize); int LoadAnyHashTable(TMPQArchive * ha); int BuildFileTable(TMPQArchive * ha); +int DefragmentFileTable(TMPQArchive * ha); +int ShrinkMalformedMpqTables(TMPQArchive * ha); + int RebuildHetTable(TMPQArchive * ha); int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize, DWORD dwNewMaxFileCount); int SaveMPQTables(TMPQArchive * ha); @@ -238,12 +244,11 @@ int SCompDecompressMpk(void * pvOutBuffer, int * pcbOutBuffer, void * pvInBuffer // Common functions - MPQ File TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry); -void * LoadMpqTable(TMPQArchive * ha, ULONGLONG ByteOffset, DWORD dwCompressedSize, DWORD dwRealSize, DWORD dwKey); +void * LoadMpqTable(TMPQArchive * ha, ULONGLONG ByteOffset, DWORD dwCompressedSize, DWORD dwRealSize, DWORD dwKey, bool * pbTableIsCut); int AllocateSectorBuffer(TMPQFile * hf); int AllocatePatchInfo(TMPQFile * hf, bool bLoadFromFile); int AllocateSectorOffsets(TMPQFile * hf, bool bLoadFromFile); int AllocateSectorChecksums(TMPQFile * hf, bool bLoadFromFile); -void CalculateRawSectorOffset(ULONGLONG & RawFilePos, TMPQFile * hf, DWORD dwSectorOffset); int WritePatchInfo(TMPQFile * hf); int WriteSectorOffsets(TMPQFile * hf); int WriteSectorChecksums(TMPQFile * hf); diff --git a/src/StormLib.h b/src/StormLib.h index c543563..af10f16 100644 --- a/src/StormLib.h +++ b/src/StormLib.h @@ -177,12 +177,14 @@ extern "C" { #define MPQ_FLAG_READ_ONLY 0x00000001 // If set, the MPQ has been open for read-only access #define MPQ_FLAG_CHANGED 0x00000002 // If set, the MPQ tables have been changed #define MPQ_FLAG_MALFORMED 0x00000004 // Malformed data structure detected (W3M map protectors) -#define MPQ_FLAG_CHECK_SECTOR_CRC 0x00000008 // Checking sector CRC when reading files -#define MPQ_FLAG_LISTFILE_INVALID 0x00000020 // If set, it means that the (listfile) has been invalidated -#define MPQ_FLAG_ATTRIBUTES_INVALID 0x00000040 // If set, it means that the (attributes) has been invalidated -#define MPQ_FLAG_SIGNATURE_INVALID 0x00000080 // If set, it means that the (signature) has been invalidated -#define MPQ_FLAG_SAVING_TABLES 0x00000100 // If set, we are saving MPQ internal files and MPQ tables -#define MPQ_FLAG_PATCH 0x00000200 // If set, this MPQ is a patch archive +#define MPQ_FLAG_HASH_TABLE_CUT 0x00000008 // The hash table goes beyond EOF +#define MPQ_FLAG_BLOCK_TABLE_CUT 0x00000010 // The hash table goes beyond EOF +#define MPQ_FLAG_CHECK_SECTOR_CRC 0x00000020 // Checking sector CRC when reading files +#define MPQ_FLAG_LISTFILE_INVALID 0x00000040 // If set, it means that the (listfile) has been invalidated +#define MPQ_FLAG_ATTRIBUTES_INVALID 0x00000080 // If set, it means that the (attributes) has been invalidated +#define MPQ_FLAG_SIGNATURE_INVALID 0x00000100 // If set, it means that the (signature) has been invalidated +#define MPQ_FLAG_SAVING_TABLES 0x00000200 // If set, we are saving MPQ internal files and MPQ tables +#define MPQ_FLAG_PATCH 0x00000400 // If set, this MPQ is a patch archive // Values for TMPQArchive::dwSubType #define MPQ_SUBTYPE_MPQ 0x00000000 // The file is a MPQ file (Blizzard games) @@ -857,6 +859,7 @@ typedef struct _TMPQArchive DWORD dwHETBlockSize; DWORD dwBETBlockSize; DWORD dwMaxFileCount; // Maximum number of files in the MPQ. Also total size of the file table. + DWORD dwHashTableSize; // Size of the hash table. Different from hash table size in the header if the hash table was shrunk DWORD dwFileTableSize; // Current size of the file table, e.g. index of the entry past the last occupied one DWORD dwReservedFiles; // Number of entries reserved for internal MPQ files (listfile, attributes) DWORD dwSectorSize; // Default size of one file sector diff --git a/test/StormTest.cpp b/test/StormTest.cpp index 1601fda..5031322 100644 --- a/test/StormTest.cpp +++ b/test/StormTest.cpp @@ -54,6 +54,8 @@ static const char * szMpqPatchDir = "1995 - Test MPQs\\patches"; typedef int (*FIND_FILE_CALLBACK)(const char * szFullPath); typedef int (*FIND_PAIR_CALLBACK)(const char * szFullPath1, const char * szFullPath2); +#define ERROR_UNDETERMINED_RESULT 0xC000FFFF + //----------------------------------------------------------------------------- // Testing data @@ -1209,8 +1211,11 @@ static int CreateFileCopy( // Close the source file FileStream_Close(pStream1); + // Create the full file name of the target file, including prefix if(szBuffer != NULL) CreateFullPathName(szBuffer, NULL, szFileCopy); + + // Report error, if any if(nError != ERROR_SUCCESS) pLogger->PrintError("Failed to create copy of MPQ"); return nError; @@ -1464,6 +1469,11 @@ static TFileData * LoadMpqFile(TLogHelper * pLogger, HANDLE hMpq, const char * s // Notify the user that we are loading a file from MPQ pLogger->PrintProgress("Loading file %s ...", GetShortPlainName(szFileName)); +#if defined(_MSC_VER) && defined(_DEBUG) +// if(!_stricmp(szFileName, "DragonSeaTurtle_Portrait.mdx")) +// DebugBreak(); +#endif + // Open the file from MPQ if(!SFileOpenFileEx(hMpq, szFileName, 0, &hFile)) nError = pLogger->PrintError("Failed to open the file %s", szFileName); @@ -1751,7 +1761,7 @@ static int OpenExistingArchive(TLogHelper * pLogger, const char * szFullPath, DW { HANDLE hMpq = NULL; TCHAR szMpqName[MAX_PATH]; - bool bReopenResult; +// bool bReopenResult; int nError = ERROR_SUCCESS; // Is it an encrypted MPQ ? @@ -1911,9 +1921,9 @@ static int AddFileToMpq( } // Check the expected error code - if(nError != nExpectedError) + if(nExpectedError != ERROR_UNDETERMINED_RESULT && nError != nExpectedError) return pLogger->PrintError("Unexpected result from SFileCreateFile(%s)", szFileName); - return ERROR_SUCCESS; + return nError; } static int AddLocalFileToMpq( @@ -2313,11 +2323,11 @@ static int TestOpenFile_OpenById(const char * szPlainName) return nError; } -// Open an empty archive (found in WoW cache - it's just a header) -static int TestOpenArchive(const char * szPlainName, const char * szListFile = NULL) +static int TestOpenArchive(const char * szPlainName, const char * szListFile = NULL, bool bDontCopyArchive = false) { TLogHelper Logger("OpenMpqTest", szPlainName); TFileData * pFileData; + const char * szCopyName = (bDontCopyArchive) ? NULL : szPlainName; HANDLE hMpq; DWORD dwFileCount = 0; DWORD dwTestFlags; @@ -2329,7 +2339,7 @@ static int TestOpenArchive(const char * szPlainName, const char * szListFile = N bIsPartialMpq = (strstr(szPlainName, ".MPQ.part") != NULL); // Copy the archive so we won't fuck up the original one - nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szPlainName, &hMpq); + nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szCopyName, &hMpq); if(nError == ERROR_SUCCESS) { // If the listfile was given, add it to the MPQ @@ -2367,6 +2377,8 @@ static int TestOpenArchive(const char * szPlainName, const char * szListFile = N return nError; } + +// Open an empty archive (found in WoW cache - it's just a header) static int TestOpenArchive_WillFail(const char * szPlainName) { TLogHelper Logger("FailMpqTest", szPlainName); @@ -2472,21 +2484,21 @@ static int TestOpenArchive_ReadOnly(const char * szPlainName, bool bReadOnly) if(nError == ERROR_SUCCESS) { nExpectedError = (bReadOnly) ? ERROR_ACCESS_DENIED : ERROR_SUCCESS; - nError = AddFileToMpq(&Logger, hMpq, "AddedFile.txt", "This is an added file.", MPQ_FILE_COMPRESS | MPQ_FILE_ENCRYPTED, 0, nExpectedError); + AddFileToMpq(&Logger, hMpq, "AddedFile.txt", "This is an added file.", MPQ_FILE_COMPRESS | MPQ_FILE_ENCRYPTED, 0, nExpectedError); } // Now try to rename a file in the MPQ. This must only succeed if the MPQ is not read only if(nError == ERROR_SUCCESS) { nExpectedError = (bReadOnly) ? ERROR_ACCESS_DENIED : ERROR_SUCCESS; - nError = RenameMpqFile(&Logger, hMpq, "spawn.mpq", "spawn-renamed.mpq", nExpectedError); + RenameMpqFile(&Logger, hMpq, "spawn.mpq", "spawn-renamed.mpq", nExpectedError); } // Now try to delete a file in the MPQ. This must only succeed if the MPQ is not read only if(nError == ERROR_SUCCESS) { nExpectedError = (bReadOnly) ? ERROR_ACCESS_DENIED : ERROR_SUCCESS; - nError = RemoveMpqFile(&Logger, hMpq, "spawn-renamed.mpq", nExpectedError); + RemoveMpqFile(&Logger, hMpq, "spawn-renamed.mpq", nExpectedError); } // Close the archive @@ -2970,9 +2982,9 @@ static int ForEachFile_OpenArchive(const char * szFullPath) return nError; } -// Adding a file to MPQ that had no (listfile) and no (attributes). -// We expect that neither of these will be present after the archive is closed -static int TestAddFile_FullArchive(const char * szFullMpq1, const char * szFullMpq2) +// Adding a file to MPQ that had size of the file table equal +// or greater than hash table, but has free entries +static int TestAddFile_FullTable(const char * szFullMpq1, const char * szFullMpq2) { TLogHelper Logger("FullMpqTest", szFullMpq1); const char * szFileName = "AddedFile001.txt"; @@ -2985,7 +2997,7 @@ static int TestAddFile_FullArchive(const char * szFullMpq1, const char * szFullM if(nError == ERROR_SUCCESS) { // Attempt to add a file - nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, MPQ_FILE_IMPLODE, MPQ_COMPRESSION_PKWARE, ERROR_DISK_FULL); + nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, MPQ_FILE_IMPLODE, MPQ_COMPRESSION_PKWARE, ERROR_SUCCESS); SFileCloseArchive(hMpq); } @@ -2994,7 +3006,7 @@ static int TestAddFile_FullArchive(const char * szFullMpq1, const char * szFullM if(nError == ERROR_SUCCESS) { // Now add a file - nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, MPQ_FILE_IMPLODE, MPQ_COMPRESSION_PKWARE, ERROR_DISK_FULL); + nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, MPQ_FILE_IMPLODE, MPQ_COMPRESSION_PKWARE, ERROR_SUCCESS); SFileCloseArchive(hMpq); } @@ -3023,6 +3035,7 @@ static int TestAddFile_ListFileTest(const char * szSourceMpq, bool bShouldHaveLi // Now add a file nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, MPQ_FILE_IMPLODE, MPQ_COMPRESSION_PKWARE); SFileCloseArchive(hMpq); + hMpq = NULL; } // Now reopen the archive @@ -3271,8 +3284,10 @@ static int TestCreateArchive_FillArchive(const char * szPlainName, DWORD dwCreat DWORD dwFlags = MPQ_FILE_ENCRYPTED | MPQ_FILE_COMPRESS; int nError; + // // Note that StormLib will round the maxfile count // up to hash table size (nearest power of two) + // if((dwCreateFlags & MPQ_CREATE_LISTFILE) == 0) dwMaxFileCount++; if((dwCreateFlags & MPQ_CREATE_ATTRIBUTES) == 0) @@ -3325,7 +3340,7 @@ static int TestCreateArchive_FillArchive(const char * szPlainName, DWORD dwCreat nError = ERROR_SUCCESS; } - // The (listfile) must be present + // The (listfile) and (attributes) must be present if(nError == ERROR_SUCCESS) { CheckIfFileIsPresent(&Logger, hMpq, LISTFILE_NAME, (dwCreateFlags & MPQ_CREATE_LISTFILE) ? true : false); @@ -3340,10 +3355,18 @@ static int TestCreateArchive_FillArchive(const char * szPlainName, DWORD dwCreat assert(nError == ERROR_SUCCESS); } - // Now add the file again. This time, it should be fail + // Now add the file again. This time, it should fail + if(nError == ERROR_SUCCESS) + { + nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, dwFlags, dwCompression, ERROR_ALREADY_EXISTS); + assert(nError != ERROR_SUCCESS); + nError = ERROR_SUCCESS; + } + + // Now add the file again. This time, it should fail if(nError == ERROR_SUCCESS) { - nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, dwFlags, dwCompression, ERROR_DISK_FULL); + nError = AddFileToMpq(&Logger, hMpq, "ShouldNotBeHere.txt", szFileData, dwFlags, dwCompression, ERROR_DISK_FULL); assert(nError != ERROR_SUCCESS); nError = ERROR_SUCCESS; } @@ -3401,7 +3424,7 @@ static int TestCreateArchive_IncMaxFileCount(const char * szPlainName) // Add one file sprintf(szFileName, "AddFile_%04u.txt", i); - nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData); + nError = AddFileToMpq(&Logger, hMpq, szFileName, szFileData, 0, 0, ERROR_UNDETERMINED_RESULT); if(nError != ERROR_SUCCESS) { // Increment the max file count by one @@ -3682,7 +3705,7 @@ static int TestCreateArchive_ListFilePos(const char * szPlainName) HANDLE hMpq = NULL; // Handle of created archive char szArchivedName[MAX_PATH]; DWORD dwMaxFileCount = 0x1E; - DWORD dwAddedCount = 0; + DWORD dwFileCount = 0; size_t i; int nError; @@ -3699,7 +3722,7 @@ static int TestCreateArchive_ListFilePos(const char * szPlainName) if(nError != ERROR_SUCCESS) break; - dwAddedCount++; + dwFileCount++; } } @@ -3712,6 +3735,8 @@ static int TestCreateArchive_ListFilePos(const char * szPlainName) nError = RemoveMpqFile(&Logger, hMpq, szArchivedName, ERROR_SUCCESS); if(nError != ERROR_SUCCESS) break; + + dwFileCount--; } } @@ -3730,7 +3755,7 @@ static int TestCreateArchive_ListFilePos(const char * szPlainName) pFileData = LoadMpqFile(&Logger, hMpq, LISTFILE_NAME); if(pFileData != NULL) { - if(pFileData->dwBlockIndex < dwAddedCount) + if(pFileData->dwBlockIndex < dwFileCount) Logger.PrintMessage("Unexpected file index of %s", LISTFILE_NAME); STORM_FREE(pFileData); } @@ -3738,20 +3763,23 @@ static int TestCreateArchive_ListFilePos(const char * szPlainName) pFileData = LoadMpqFile(&Logger, hMpq, ATTRIBUTES_NAME); if(pFileData != NULL) { - if(pFileData->dwBlockIndex <= dwAddedCount) + if(pFileData->dwBlockIndex <= dwFileCount) Logger.PrintMessage("Unexpected file index of %s", ATTRIBUTES_NAME); STORM_FREE(pFileData); } - // Add new file to the archive. It should be added to position 0 - // (since position 0 should be free) + // Add new file to the archive. It should be added to the last position nError = AddFileToMpq(&Logger, hMpq, szReaddedFile, "This is a re-added file.", 0, 0, ERROR_SUCCESS); if(nError == ERROR_SUCCESS) { + // Force update of the tables + SFileFlushArchive(hMpq); + + // Load the file pFileData = LoadMpqFile(&Logger, hMpq, szReaddedFile); if(pFileData != NULL) { - if(pFileData->dwBlockIndex != 0) + if(pFileData->dwBlockIndex != dwFileCount) Logger.PrintMessage("Unexpected file index of %s", szReaddedFile); STORM_FREE(pFileData); } @@ -4075,10 +4103,14 @@ int main(int argc, char * argv[]) if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_BOBA.w3m"); */ - // Open an Warcraft III map locked by the BOBA protector + // Open an Warcraft III map locked by a protector + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("MPQ_2015_v1_ProtectedMap_KangTooJee.w3x"); + + // Open an Warcraft III map locked by a protector if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2015_v1_ProtectedMap_Somj2hM16.w3x"); -/* + // Open an Warcraft III map whose "(attributes)" file has (BlockTableSize-1) entries if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2014_v1_AttributesOneEntryLess.w3x"); @@ -4109,7 +4141,7 @@ int main(int argc, char * argv[]) // Open an archive that is merged with multiple files if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("blk4-file://streaming/model.MPQ.0"); + nError = TestOpenArchive("blk4-file://streaming/model.MPQ.0", NULL, true); // Open every MPQ that we have in the storage if(nError == ERROR_SUCCESS) @@ -4185,9 +4217,9 @@ int main(int argc, char * argv[]) // if(nError == ERROR_SUCCESS) // nError = TestOpenArchive_CompactingTest("MPQ_2014_v1_CompactTest.w3x", "ListFile_Blizzard.txt"); -*/ + if(nError == ERROR_SUCCESS) - nError = TestAddFile_FullArchive("MPQ_2014_v1_out1.w3x", "MPQ_2014_v1_out2.w3x"); + nError = TestAddFile_FullTable("MPQ_2014_v1_out1.w3x", "MPQ_2014_v1_out2.w3x"); // Test modifying file with no (listfile) and no (attributes) if(nError == ERROR_SUCCESS) |