diff options
author | unknown <E:\Ladik\Mail> | 2015-04-21 19:36:13 +0200 |
---|---|---|
committer | unknown <E:\Ladik\Mail> | 2015-04-21 19:36:13 +0200 |
commit | 5747ad4910966b27b4da3f0d0e5f085697e5bf7b (patch) | |
tree | 5e003449eec18c2f99d10dda2796b77a43b5e86c /src | |
parent | 68e2c2527db10dcfdd9fdc2219c7fe9784abf738 (diff) |
+ Defragmenting file table fixed
Diffstat (limited to 'src')
-rw-r--r-- | src/FileStream.cpp | 48 | ||||
-rw-r--r-- | src/SBaseCommon.cpp | 2 | ||||
-rw-r--r-- | src/SBaseFileTable.cpp | 123 | ||||
-rw-r--r-- | src/SFileGetFileInfo.cpp | 2 | ||||
-rw-r--r-- | src/SFileListFile.cpp | 3 | ||||
-rw-r--r-- | src/StormLib.h | 2 |
6 files changed, 85 insertions, 95 deletions
diff --git a/src/FileStream.cpp b/src/FileStream.cpp index 2f6e599..53740fa 100644 --- a/src/FileStream.cpp +++ b/src/FileStream.cpp @@ -2888,51 +2888,3 @@ void CopyFileName(char * szTarget, const TCHAR * szSource, size_t cchLength) szTarget[cchLength] = 0; } #endif - -//----------------------------------------------------------------------------- -// main - for testing purposes - -#ifdef __STORMLIB_TEST__ -int FileStream_Test(const TCHAR * szFileName, DWORD dwStreamFlags) -{ - TFileStream * pStream; - TMPQHeader MpqHeader; - ULONGLONG FilePos; - TMPQBlock * pBlock; - TMPQHash * pHash; - - InitializeMpqCryptography(); - - pStream = FileStream_OpenFile(szFileName, dwStreamFlags); - if(pStream == NULL) - return GetLastError(); - - // Read the MPQ header - FileStream_Read(pStream, NULL, &MpqHeader, MPQ_HEADER_SIZE_V2); - if(MpqHeader.dwID != ID_MPQ) - return ERROR_FILE_CORRUPT; - - // Read the hash table - pHash = STORM_ALLOC(TMPQHash, MpqHeader.dwHashTableSize); - if(pHash != NULL) - { - FilePos = MpqHeader.dwHashTablePos; - FileStream_Read(pStream, &FilePos, pHash, MpqHeader.dwHashTableSize * sizeof(TMPQHash)); - DecryptMpqBlock(pHash, MpqHeader.dwHashTableSize * sizeof(TMPQHash), MPQ_KEY_HASH_TABLE); - STORM_FREE(pHash); - } - - // Read the block table - pBlock = STORM_ALLOC(TMPQBlock, MpqHeader.dwBlockTableSize); - if(pBlock != NULL) - { - FilePos = MpqHeader.dwBlockTablePos; - FileStream_Read(pStream, &FilePos, pBlock, MpqHeader.dwBlockTableSize * sizeof(TMPQBlock)); - DecryptMpqBlock(pBlock, MpqHeader.dwBlockTableSize * sizeof(TMPQBlock), MPQ_KEY_BLOCK_TABLE); - STORM_FREE(pBlock); - } - - FileStream_Close(pStream); - return ERROR_SUCCESS; -} -#endif diff --git a/src/SBaseCommon.cpp b/src/SBaseCommon.cpp index 9509b6f..062a737 100644 --- a/src/SBaseCommon.cpp +++ b/src/SBaseCommon.cpp @@ -612,7 +612,7 @@ TMPQHash * GetNextHashEntry(TMPQArchive * ha, TMPQHash * pFirstHash, TMPQHash * pHash = ha->pHashTable + dwIndex; // If the entry matches, we found it. - if(pHash->dwName1 == dwName1 && pHash->dwName2 == dwName2 && pHash->dwBlockIndex < ha->pHeader->dwBlockTableSize) + if(pHash->dwName1 == dwName1 && pHash->dwName2 == dwName2 && pHash->dwBlockIndex < ha->dwFileTableSize) return pHash; // If that hash entry is a free entry, it means we haven't found the file diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp index 7209de2..97ecb53 100644 --- a/src/SBaseFileTable.cpp +++ b/src/SBaseFileTable.cpp @@ -568,18 +568,14 @@ int ConvertMpqHeaderToFormat4( //----------------------------------------------------------------------------- // Support for hash table -static DWORD IsValidHashEntry(TMPQArchive * ha, TMPQHash * pHash) +static bool IsValidHashEntry(TMPQArchive * ha, TMPQHash * pHash) { // The block table index must be smaller than file table size - if(pHash->dwBlockIndex < ha->dwFileTableSize) - { - // The block table entry must contain a file - if(ha->pFileTable[pHash->dwBlockIndex].dwFlags & MPQ_FILE_EXISTS) - return pHash->dwBlockIndex; - } - - // Return an invalid index - return HASH_ENTRY_FREE; + // The block table entry's flags must have MPQ_FILE_EXISTS set + return ( + (pHash->dwBlockIndex < ha->dwFileTableSize) && + (ha->pFileTable[pHash->dwBlockIndex].dwFlags & MPQ_FILE_EXISTS_MASK) == MPQ_FILE_EXISTS + ); } // Returns a hash table entry in the following order: @@ -685,7 +681,8 @@ static int BuildFileTableFromBlockTable( // a6d79af0 e61a0932 0000d761 0000dacb <== Fake valid // a6d79af0 e61a0932 00000000 0000002f <== Real file entry (block index is valid) // a6d79af0 e61a0932 00005a4f 000093bc <== Fake valid - // + // + if(pHash->dwBlockIndex < pHeader->dwBlockTableSize) { // Get the pointer to the file entry and the block entry @@ -697,7 +694,7 @@ static int BuildFileTableFromBlockTable( { // ByteOffset is only valid if file size is not zero pFileEntry->ByteOffset = pBlock->dwFilePos; - if(pFileEntry->ByteOffset == 0 && pBlock->dwCSize == 0) + if(pFileEntry->ByteOffset == 0 && pBlock->dwFSize == 0) pFileEntry->ByteOffset = ha->pHeader->dwHeaderSize; pFileEntry->dwHashIndex = (DWORD)(pHash - ha->pHashTable); @@ -706,16 +703,7 @@ static int BuildFileTableFromBlockTable( pFileEntry->dwFlags = pBlock->dwFlags; pFileEntry->lcLocale = pHash->lcLocale; pFileEntry->wPlatform = pHash->wPlatform; - continue; } - - // If the hash table entry doesn't point to the valid file item, - // we invalidate the hash table entry - pHash->dwName1 = 0xFFFFFFFF; - pHash->dwName2 = 0xFFFFFFFF; - pHash->lcLocale = 0xFFFF; - pHash->wPlatform = 0xFFFF; - pHash->dwBlockIndex = HASH_ENTRY_DELETED; } } @@ -2470,12 +2458,11 @@ int BuildFileTable(TMPQArchive * ha) int DefragmentFileTable(TMPQArchive * ha) { TFileEntry * pNewTable; - TMPQHash * pHashEnd = ha->pHashTable + ha->pHeader->dwHashTableSize; + TMPQHash * pHashEnd = ha->pHashTable + ha->dwHashTableSize; TMPQHash * pHash; PDWORD NewIndexes; DWORD dwNewTableSize = 0; DWORD dwNewBlockIndex; - DWORD dwBlockIndex = 0; // Allocate brand new file table NewIndexes = STORM_ALLOC(DWORD, ha->dwFileTableSize); @@ -2488,34 +2475,37 @@ int DefragmentFileTable(TMPQArchive * ha) for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) { // If the entry is valid, move it - if((dwBlockIndex = IsValidHashEntry(ha, pHash)) != HASH_ENTRY_FREE) + if(IsValidHashEntry(ha, pHash)) { // Was the new index already resolved? - // Note: More hash entries can point to the same file - if(NewIndexes[dwBlockIndex] == 0xFFFFFFFF) - NewIndexes[dwBlockIndex] = dwNewTableSize++; + // Note: Multiple hash table entries can point to the same block table entry + if(NewIndexes[pHash->dwBlockIndex] == 0xFFFFFFFF) + NewIndexes[pHash->dwBlockIndex] = dwNewTableSize++; } } + // We have to append reserved files + dwNewTableSize += ha->dwReservedFiles; + // Allocate new file table - pNewTable = STORM_ALLOC(TFileEntry, dwNewTableSize); + pNewTable = STORM_ALLOC(TFileEntry, ha->dwMaxFileCount); if(pNewTable != NULL) { // Fill the new table with zeros - memset(pNewTable, 0, sizeof(TFileEntry) * dwNewTableSize); + memset(pNewTable, 0, sizeof(TFileEntry) * ha->dwMaxFileCount); // Create map of OldBlockIndex => NewBlockIndex for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) { // If the entry is valid, move it - if((dwBlockIndex = IsValidHashEntry(ha, pHash)) != HASH_ENTRY_FREE) + if(IsValidHashEntry(ha, pHash)) { // The new index should be resolved by now - assert(NewIndexes[dwBlockIndex] != 0xFFFFFFFF); - dwNewBlockIndex = NewIndexes[dwBlockIndex]; + assert(NewIndexes[pHash->dwBlockIndex] != 0xFFFFFFFF); + dwNewBlockIndex = NewIndexes[pHash->dwBlockIndex]; // Remap the old entry to the new entry - pNewTable[dwNewBlockIndex] = ha->pFileTable[dwBlockIndex]; + pNewTable[dwNewBlockIndex] = ha->pFileTable[pHash->dwBlockIndex]; pHash->dwBlockIndex = dwNewBlockIndex; } } @@ -2533,24 +2523,67 @@ int DefragmentFileTable(TMPQArchive * ha) return ERROR_SUCCESS; } +// Defragment the file table so it does not contain any gaps +// Note: As long as all values of all TMPQHash::dwBlockIndex +// are not HASH_ENTRY_FREE, the startup search index does not matter. +// Hash table is circular, so as long as there is no terminator, +// all entries will be found. +int DefragmentHashTable(TMPQArchive * ha) +{ + TMPQHash * pNewTable; + TMPQHash * pHashEnd = ha->pHashTable + ha->pHeader->dwHashTableSize; + TMPQHash * pNewHash; + TMPQHash * pHash; + DWORD dwNewTableSize = 0; + DWORD dwNewIndex = 0; + + // Calculate how many entries in the hash table we really need + for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) + dwNewTableSize += IsValidHashEntry(ha, pHash) ? 1 : 0; + dwNewTableSize = GetHashTableSizeForFileCount(dwNewTableSize); + + // Could the hash table be shrinked? + if(dwNewTableSize < ha->dwHashTableSize) + { + pNewTable = STORM_ALLOC(TMPQHash, dwNewTableSize); + if(pNewTable != NULL) + { + // Initialize the new table + memset(pNewTable, 0xFF, dwNewTableSize * sizeof(TMPQHash)); + pNewHash = pNewTable; + + // Copy the entries from the current hash table to new hash table + for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) + { + if(IsValidHashEntry(ha, pHash)) + { + assert(dwNewIndex < dwNewTableSize); + pNewHash[dwNewIndex++] = pHash[0]; + } + } + + // Fill the remaining entries so they look like deleted + while(dwNewIndex < dwNewTableSize) + pNewHash[dwNewIndex++].dwBlockIndex = HASH_ENTRY_DELETED; + + // Swap the hash tables + STORM_FREE(ha->pHashTable); + ha->pHashTable = pNewTable; + ha->dwHashTableSize = dwNewTableSize; + } + } + return ERROR_SUCCESS; +} + // Performs final validation (and possible defragmentation) // of malformed hash+block tables int ShrinkMalformedMpqTables(TMPQArchive * ha) { - TFileEntry * pFileTable = ha->pFileTable; - TMPQHash * pHashTable = ha->pHashTable; - DWORD i; - // Sanity checks assert(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1); assert(ha->pHashTable != NULL); assert(ha->pFileTable != NULL); - // - // Note: Unhandled possible case where multiple hash table entries - // point to the same block table entry. - // - // Now perform the defragmentation of the block table if(ha->dwFlags & MPQ_FLAG_BLOCK_TABLE_CUT) { @@ -2562,6 +2595,10 @@ int ShrinkMalformedMpqTables(TMPQArchive * ha) // This will greatly improve performance on searching and loading listfile if(ha->dwFlags & MPQ_FLAG_HASH_TABLE_CUT) { + assert(ha->dwHashTableSize == ha->pHeader->dwHashTableSize); + DefragmentHashTable(ha); + } +/* TMPQHash * pSrcEntry = ha->pHashTable; TMPQHash * pTrgEntry = ha->pHashTable; DWORD dwNewEntryCount; @@ -2601,7 +2638,7 @@ int ShrinkMalformedMpqTables(TMPQArchive * ha) // Change the hash table size in the header ha->dwHashTableSize = dwNewTableSize; } - +*/ return ERROR_SUCCESS; } diff --git a/src/SFileGetFileInfo.cpp b/src/SFileGetFileInfo.cpp index d6d6da3..f7faea0 100644 --- a/src/SFileGetFileInfo.cpp +++ b/src/SFileGetFileInfo.cpp @@ -376,7 +376,7 @@ bool WINAPI SFileGetFileInfo( if(ha != NULL && ha->pHashTable != NULL) { pvSrcFileInfo = ha->pHashTable; - cbSrcFileInfo = ha->pHeader->dwHashTableSize * sizeof(TMPQHash); + cbSrcFileInfo = ha->dwHashTableSize * sizeof(TMPQHash); nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER; } break; diff --git a/src/SFileListFile.cpp b/src/SFileListFile.cpp index f5251bc..5f8a9df 100644 --- a/src/SFileListFile.cpp +++ b/src/SFileListFile.cpp @@ -382,7 +382,6 @@ static LPBYTE CreateListFile(TMPQArchive * ha, DWORD * pcbListFile) // If the file name is already there, does nothing. static int SListFileCreateNodeForAllLocales(TMPQArchive * ha, const char * szFileName) { - TMPQHeader * pHeader = ha->pHeader; TFileEntry * pFileEntry; TMPQHash * pFirstHash; TMPQHash * pHash; @@ -410,7 +409,7 @@ static int SListFileCreateNodeForAllLocales(TMPQArchive * ha, const char * szFil while(pHash != NULL) { // Is it a valid file table index ? - if(pHash->dwBlockIndex < pHeader->dwBlockTableSize) + if(pHash->dwBlockIndex < ha->dwFileTableSize) { // Allocate file name for the file entry AllocateFileName(ha, ha->pFileTable + pHash->dwBlockIndex, szFileName); diff --git a/src/StormLib.h b/src/StormLib.h index e523b3e..3d4fddf 100644 --- a/src/StormLib.h +++ b/src/StormLib.h @@ -212,6 +212,8 @@ extern "C" { #define MPQ_FILE_EXISTS 0x80000000 // Set if file exists, reset when the file was deleted #define MPQ_FILE_REPLACEEXISTING 0x80000000 // Replace when the file exist (SFileAddFile) +#define MPQ_FILE_EXISTS_MASK 0xF00000FF // These must be either zero or MPQ_FILE_EXISTS + #define MPQ_FILE_VALID_FLAGS (MPQ_FILE_IMPLODE | \ MPQ_FILE_COMPRESS | \ MPQ_FILE_ENCRYPTED | \ |