diff options
| -rw-r--r-- | StormLib_test.vcproj | 2 | ||||
| -rw-r--r-- | src/SBaseCommon.cpp | 54 | ||||
| -rw-r--r-- | src/SBaseDumpData.cpp | 25 | ||||
| -rw-r--r-- | src/SBaseFileTable.cpp | 25 | ||||
| -rw-r--r-- | src/SFileAddFile.cpp | 174 | ||||
| -rw-r--r-- | src/SFileAttributes.cpp | 3 | ||||
| -rw-r--r-- | src/SFileCompactArchive.cpp | 144 | ||||
| -rw-r--r-- | src/SFileListFile.cpp | 16 | ||||
| -rw-r--r-- | src/SFileReadFile.cpp | 12 | ||||
| -rw-r--r-- | src/StormCommon.h | 17 | ||||
| -rw-r--r-- | test/StormTest.cpp | 163 |
11 files changed, 431 insertions, 204 deletions
diff --git a/StormLib_test.vcproj b/StormLib_test.vcproj index 09cef44..7bb6ea3 100644 --- a/StormLib_test.vcproj +++ b/StormLib_test.vcproj @@ -51,7 +51,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="./src/libtomcrypt/src/headers" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;__STORMLIB_TEST__" + PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;__STORMLIB_TEST__;__STORMLIB_DUMP_DATA__" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="1" diff --git a/src/SBaseCommon.cpp b/src/SBaseCommon.cpp index b4d62e0..22b1221 100644 --- a/src/SBaseCommon.cpp +++ b/src/SBaseCommon.cpp @@ -226,7 +226,7 @@ ULONGLONG HashStringJenkins(const char * szFileName) { LPBYTE pbFileName = (LPBYTE)szFileName; char * szTemp; - char szLocFileName[0x108]; + char szNameBuff[0x108]; size_t nLength = 0; unsigned int primary_hash = 1; unsigned int secondary_hash = 2; @@ -234,21 +234,22 @@ ULONGLONG HashStringJenkins(const char * szFileName) // Normalize the file name - convert to uppercase, and convert "/" to "\\". if(pbFileName != NULL) { - szTemp = szLocFileName; - while(*pbFileName != 0) - *szTemp++ = (char)AsciiToLowerTable[*pbFileName++]; - *szTemp = 0; + char * szNamePtr = szNameBuff; + char * szNameEnd = szNamePtr + sizeof(szNameBuff); - nLength = szTemp - szLocFileName; + // Normalize the file name. Doesn't have to be zero terminated for hashing + while(szNamePtr < szNameEnd && pbFileName[0] != 0) + *szNamePtr++ = (char)AsciiToLowerTable[*pbFileName++]; + nLength = szNamePtr - szNameBuff; } // Thanks Quantam for finding out what the algorithm is. // I am really getting old for reversing large chunks of assembly // that does hashing :-) - hashlittle2(szLocFileName, nLength, &secondary_hash, &primary_hash); + hashlittle2(szNameBuff, nLength, &secondary_hash, &primary_hash); // Combine those 2 together - return (ULONGLONG)primary_hash * (ULONGLONG)0x100000000ULL + (ULONGLONG)secondary_hash; + return ((ULONGLONG)primary_hash << 0x20) | (ULONGLONG)secondary_hash; } //----------------------------------------------------------------------------- @@ -722,6 +723,43 @@ TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry) return hf; } +TMPQFile * CreateWritableHandle(TMPQArchive * ha, DWORD dwFileSize) +{ + ULONGLONG FreeMpqSpace; + ULONGLONG TempPos; + TMPQFile * hf; + + // We need to find the position in the MPQ where we save the file data + FreeMpqSpace = FindFreeMpqSpace(ha); + + // When format V1, the size of the archive cannot exceed 4 GB + if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) + { + TempPos = FreeMpqSpace + + dwFileSize + + (ha->pHeader->dwHashTableSize * sizeof(TMPQHash)) + + (ha->dwFileTableSize * sizeof(TMPQBlock)); + if((TempPos >> 32) != 0) + { + SetLastError(ERROR_DISK_FULL); + return NULL; + } + } + + // Allocate the file handle + hf = CreateFileHandle(ha, NULL); + if(hf == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return NULL; + } + + // We need to find the position in the MPQ where we save the file data + hf->MpqFilePos = FreeMpqSpace; + hf->bIsWriteHandle = true; + return hf; +} + // Loads a table from MPQ. // Can be used for hash table, block table, sector offset table or sector checksum table void * LoadMpqTable( diff --git a/src/SBaseDumpData.cpp b/src/SBaseDumpData.cpp index d2b1206..d156030 100644 --- a/src/SBaseDumpData.cpp +++ b/src/SBaseDumpData.cpp @@ -161,4 +161,29 @@ void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable) printf("-----------------------------------------------------------------------------------------\n"); } +void DumpFileTable(TFileEntry * pFileTable, DWORD dwFileTableSize) +{ + DWORD i; + + if(pFileTable == NULL || dwFileTableSize == 0) + return; + + printf("== File Table =================================\n"); + for(i = 0; i < dwFileTableSize; i++, pFileTable++) + { + printf("[%04u] %08X-%08X %08X-%08X %08X-%08X 0x%08X 0x%08X 0x%08X %s\n", i, + (DWORD)(pFileTable->FileNameHash >> 0x20), + (DWORD)(pFileTable->FileNameHash & 0xFFFFFFFF), + (DWORD)(pFileTable->ByteOffset >> 0x20), + (DWORD)(pFileTable->ByteOffset & 0xFFFFFFFF), + (DWORD)(pFileTable->FileTime >> 0x20), + (DWORD)(pFileTable->FileTime & 0xFFFFFFFF), + pFileTable->dwFileSize, + pFileTable->dwCmpSize, + pFileTable->dwFlags, + pFileTable->szFileName != NULL ? pFileTable->szFileName : ""); + } + printf("-----------------------------------------------\n\n"); +} + #endif // __STORMLIB_DUMP_DATA__ diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp index 3e98ebb..5d7d973 100644 --- a/src/SBaseFileTable.cpp +++ b/src/SBaseFileTable.cpp @@ -377,6 +377,11 @@ int ConvertMpqHeaderToFormat4( if(pHeader->dwBlockTablePos <= pHeader->dwHeaderSize || (pHeader->dwBlockTablePos & 0x80000000)) ha->dwFlags |= MPQ_FLAG_MALFORMED; + // Only low byte of sector size is really used + if(pHeader->wSectorSize & 0xFF00) + ha->dwFlags |= MPQ_FLAG_MALFORMED; + pHeader->wSectorSize = pHeader->wSectorSize & 0xFF; + // Fill the rest of the header memset((LPBYTE)pHeader + MPQ_HEADER_SIZE_V1, 0, sizeof(TMPQHeader) - MPQ_HEADER_SIZE_V1); pHeader->BlockTableSize64 = pHeader->dwBlockTableSize * sizeof(TMPQBlock); @@ -569,6 +574,13 @@ int ConvertMpqHeaderToFormat4( // Support for hash table // Hash entry verification when the file table does not exist yet +bool IsValidHashEntry(TMPQArchive * ha, TMPQHash * pHash) +{ + TFileEntry * pFileEntry = ha->pFileTable + pHash->dwBlockIndex; + return ((pHash->dwBlockIndex < ha->dwFileTableSize) && (pFileEntry->dwFlags & MPQ_FILE_EXISTS)) ? true : false; +} + +// Hash entry verification when the file table does not exist yet static bool IsValidHashEntry1(TMPQArchive * ha, TMPQHash * pHash, TMPQBlock * pBlockTable) { ULONGLONG ByteOffset; @@ -592,13 +604,6 @@ static bool IsValidHashEntry1(TMPQArchive * ha, TMPQHash * pHash, TMPQBlock * pB return false; } -// Hash entry verification when the file table does not exist yet -static bool IsValidHashEntry2(TMPQArchive * ha, TMPQHash * pHash) -{ - TFileEntry * pFileEntry = ha->pFileTable + pHash->dwBlockIndex; - return ((pHash->dwBlockIndex < ha->dwFileTableSize) && (pFileEntry->dwFlags & MPQ_FILE_EXISTS)) ? true : false; -} - // Returns a hash table entry in the following order: // 1) A hash table entry with the preferred locale // 2) A hash table entry with the neutral locale @@ -704,6 +709,7 @@ static TMPQHash * DefragmentHashTable( if(dwNewTableSize < pHeader->dwHashTableSize) { pHashTable = STORM_REALLOC(TMPQHash, pHashTable, dwNewTableSize); + ha->pHeader->BlockTableSize64 = dwNewTableSize * sizeof(TMPQHash); ha->pHeader->dwHashTableSize = dwNewTableSize; } @@ -814,10 +820,13 @@ static int BuildFileTableFromBlockTable( if(ha->dwFileTableSize > ha->dwMaxFileCount) { ha->pFileTable = STORM_REALLOC(TFileEntry, ha->pFileTable, ha->dwMaxFileCount); + ha->pHeader->BlockTableSize64 = ha->dwMaxFileCount * sizeof(TMPQBlock); ha->pHeader->dwBlockTableSize = ha->dwMaxFileCount; ha->dwFileTableSize = ha->dwMaxFileCount; } +// DumpFileTable(ha->pFileTable, ha->dwFileTableSize); + // Free the translation table STORM_FREE(DefragmentTable); } @@ -2697,7 +2706,7 @@ int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize) // Parse the old hash table and copy all entries to the new table for(pHash = pOldHashTable; pHash < pHashTableEnd; pHash++) { - if(IsValidHashEntry2(ha, pHash)) + if(IsValidHashEntry(ha, pHash)) { pFileEntry = ha->pFileTable + pHash->dwBlockIndex; AllocateHashEntry(ha, pFileEntry, pHash->lcLocale); diff --git a/src/SFileAddFile.cpp b/src/SFileAddFile.cpp index 70a857b..ea3c5f9 100644 --- a/src/SFileAddFile.cpp +++ b/src/SFileAddFile.cpp @@ -80,6 +80,45 @@ static bool IsWaveFile_16BitsPerAdpcmSample( return false; } +static int FillWritableHandle( + TMPQArchive * ha, + TMPQFile * hf, + ULONGLONG FileTime, + DWORD dwFileSize, + DWORD dwFlags) +{ + TFileEntry * pFileEntry = hf->pFileEntry; + + // Initialize the hash entry for the file + hf->RawFilePos = ha->MpqPos + hf->MpqFilePos; + hf->dwDataSize = dwFileSize; + + // Initialize the block table entry for the file + pFileEntry->ByteOffset = hf->MpqFilePos; + pFileEntry->dwFileSize = dwFileSize; + pFileEntry->dwCmpSize = 0; + pFileEntry->dwFlags = dwFlags | MPQ_FILE_EXISTS; + + // Initialize the file time, CRC32 and MD5 + assert(sizeof(hf->hctx) >= sizeof(hash_state)); + memset(pFileEntry->md5, 0, MD5_DIGEST_SIZE); + md5_init((hash_state *)hf->hctx); + pFileEntry->dwCrc32 = crc32(0, Z_NULL, 0); + + // If the caller gave us a file time, use it. + pFileEntry->FileTime = FileTime; + + // Mark the archive as modified + ha->dwFlags |= MPQ_FLAG_CHANGED; + + // Call the callback, if needed + if(ha->pfnAddFileCB != NULL) + ha->pfnAddFileCB(ha->pvAddFileUserData, 0, hf->dwDataSize, false); + hf->nAddFileError = ERROR_SUCCESS; + + return ERROR_SUCCESS; +} + //----------------------------------------------------------------------------- // MPQ write data functions @@ -366,7 +405,6 @@ int SFileAddFile_Init( TMPQFile ** phf) { TFileEntry * pFileEntry = NULL; - ULONGLONG TempPos; // For various file offset calculations TMPQFile * hf = NULL; // File structure for newly added file DWORD dwHashIndex = HASH_ENTRY_FREE; int nError = ERROR_SUCCESS; @@ -395,28 +433,9 @@ int SFileAddFile_Init( lcLocale = 0; // Allocate the TMPQFile entry for newly added file - hf = CreateFileHandle(ha, NULL); + hf = CreateWritableHandle(ha, dwFileSize); if(hf == NULL) - nError = ERROR_NOT_ENOUGH_MEMORY; - - // Find a free space in the MPQ and verify if it's not over 4 GB on MPQs v1 - if(nError == ERROR_SUCCESS) - { - // Find the position where the file will be stored - hf->MpqFilePos = FindFreeMpqSpace(ha); - hf->RawFilePos = ha->MpqPos + hf->MpqFilePos; - hf->bIsWriteHandle = true; - - // When format V1, the size of the archive cannot exceed 4 GB - if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1) - { - TempPos = hf->MpqFilePos + dwFileSize; - TempPos += ha->pHeader->dwHashTableSize * sizeof(TMPQHash); - TempPos += ha->dwFileTableSize * sizeof(TMPQBlock); - if((TempPos >> 32) != 0) - nError = ERROR_DISK_FULL; - } - } + nError = GetLastError(); // Allocate file entry in the MPQ if(nError == ERROR_SUCCESS) @@ -440,6 +459,24 @@ int SFileAddFile_Init( if(pFileEntry == NULL) nError = ERROR_DISK_FULL; } + + // Set the file entry to the file structure + hf->pFileEntry = pFileEntry; + } + + // Prepare the pointer to hash table entry + if(nError == ERROR_SUCCESS && ha->pHashTable != NULL && dwHashIndex < ha->pHeader->dwHashTableSize) + { + hf->pHashEntry = ha->pHashTable + dwHashIndex; + hf->pHashEntry->lcLocale = (USHORT)lcLocale; + } + + // Prepare the file key + if(nError == ERROR_SUCCESS && (dwFlags & MPQ_FILE_ENCRYPTED)) + { + hf->dwFileKey = DecryptFileKey(szFileName, hf->MpqFilePos, dwFileSize, dwFlags); + if(hf->dwFileKey == 0) + nError = ERROR_UNKNOWN_FILE_KEY; } // Fill the file entry and TMPQFile structure @@ -449,46 +486,77 @@ int SFileAddFile_Init( assert(pFileEntry->szFileName != NULL); assert(_stricmp(pFileEntry->szFileName, szFileName) == 0); - // Initialize the hash entry for the file - hf->pFileEntry = pFileEntry; - hf->dwDataSize = dwFileSize; + nError = FillWritableHandle(ha, hf, FileTime, dwFileSize, dwFlags); + } - // Set the hash table entry - if(ha->pHashTable != NULL && dwHashIndex < ha->pHeader->dwHashTableSize) - { - hf->pHashEntry = ha->pHashTable + dwHashIndex; - hf->pHashEntry->lcLocale = (USHORT)lcLocale; - } + // Free the file handle if failed + if(nError != ERROR_SUCCESS && hf != NULL) + FreeFileHandle(hf); + + // Give the handle to the caller + *phf = hf; + return nError; +} + +int SFileAddFile_Init( + TMPQArchive * ha, + TMPQFile * hfSrc, + TMPQFile ** phf) +{ + TFileEntry * pFileEntry = NULL; + TMPQFile * hf = NULL; // File structure for newly added file + ULONGLONG FileTime = hfSrc->pFileEntry->FileTime; + DWORD dwFileSize = hfSrc->pFileEntry->dwFileSize; + DWORD dwFlags = hfSrc->pFileEntry->dwFlags; + int nError = ERROR_SUCCESS; + + // Allocate the TMPQFile entry for newly added file + hf = CreateWritableHandle(ha, dwFileSize); + if(hf == NULL) + nError = ERROR_NOT_ENOUGH_MEMORY; - // Decrypt the file key - if(dwFlags & MPQ_FILE_ENCRYPTED) - hf->dwFileKey = DecryptFileKey(szFileName, hf->MpqFilePos, dwFileSize, dwFlags); + // We need to keep the file entry index the same like in the source archive + // This is because multiple hash table entries can point to the same file entry + if(nError == ERROR_SUCCESS) + { + // Retrieve the file entry for the target file + pFileEntry = ha->pFileTable + (hfSrc->pFileEntry - hfSrc->ha->pFileTable); - // Initialize the block table entry for the file - pFileEntry->ByteOffset = hf->MpqFilePos; - pFileEntry->dwFileSize = dwFileSize; - pFileEntry->dwCmpSize = 0; - pFileEntry->dwFlags = dwFlags | MPQ_FILE_EXISTS; + // Copy all variables except file name + if((pFileEntry->dwFlags & MPQ_FILE_EXISTS) == 0) + { + pFileEntry[0] = hfSrc->pFileEntry[0]; + pFileEntry->szFileName = NULL; + } + else + nError = ERROR_ALREADY_EXISTS; - // Initialize the file time, CRC32 and MD5 - assert(sizeof(hf->hctx) >= sizeof(hash_state)); - memset(pFileEntry->md5, 0, MD5_DIGEST_SIZE); - md5_init((hash_state *)hf->hctx); - pFileEntry->dwCrc32 = crc32(0, Z_NULL, 0); + // Set the file entry to the file structure + hf->pFileEntry = pFileEntry; + } - // If the caller gave us a file time, use it. - pFileEntry->FileTime = FileTime; + // Prepare the pointer to hash table entry + if(nError == ERROR_SUCCESS && ha->pHashTable != NULL && hfSrc->pHashEntry != NULL) + { + hf->dwHashIndex = (DWORD)(hfSrc->pHashEntry - hfSrc->ha->pHashTable); + hf->pHashEntry = ha->pHashTable + hf->dwHashIndex; + } - // Mark the archive as modified - ha->dwFlags |= MPQ_FLAG_CHANGED; + // Prepare the file key (copy from source file) + if(nError == ERROR_SUCCESS && (dwFlags & MPQ_FILE_ENCRYPTED)) + { + hf->dwFileKey = hfSrc->dwFileKey; + if(hf->dwFileKey == 0) + nError = ERROR_UNKNOWN_FILE_KEY; + } - // Call the callback, if needed - if(ha->pfnAddFileCB != NULL) - ha->pfnAddFileCB(ha->pvAddFileUserData, 0, hf->dwDataSize, false); - hf->nAddFileError = ERROR_SUCCESS; + // Fill the file entry and TMPQFile structure + if(nError == ERROR_SUCCESS) + { + nError = FillWritableHandle(ha, hf, FileTime, dwFileSize, dwFlags); } - // Fre the file handle if failed + // Free the file handle if failed if(nError != ERROR_SUCCESS && hf != NULL) FreeFileHandle(hf); diff --git a/src/SFileAttributes.cpp b/src/SFileAttributes.cpp index bd1956d..d37d83c 100644 --- a/src/SFileAttributes.cpp +++ b/src/SFileAttributes.cpp @@ -424,9 +424,6 @@ int SAttrFileSaveToMpq(TMPQArchive * ha) pbAttrFile = CreateAttributesFile(ha, &cbAttrFile); if(pbAttrFile != NULL) { - // We expect it to be nonzero size - assert(cbAttrFile != 0); - // Determine the real flags for (attributes) if(ha->dwFileFlags2 == MPQ_FILE_EXISTS) ha->dwFileFlags2 = GetDefaultSpecialFileFlags(cbAttrFile, ha->pHeader->wFormatVersion); diff --git a/src/SFileCompactArchive.cpp b/src/SFileCompactArchive.cpp index 2c0ef5a..be1ca66 100644 --- a/src/SFileCompactArchive.cpp +++ b/src/SFileCompactArchive.cpp @@ -81,32 +81,7 @@ static int CheckIfAllKeysKnown(TMPQArchive * ha, const char * szListFile, LPDWOR pFileEntry->dwFlags); continue; } -/* - // If the file has a nonzero size, we can try to read few bytes of data - // and force to detect the decryption key that way - if(pFileEntry->dwFileSize > 0x10) - { - TMPQFile * hf = NULL; - DWORD dwBytesRead = 0; - DWORD FileData[4]; - - // Create file handle where we load the sector offset table - hf = CreateFileHandle(ha, pFileEntry); - if(hf != NULL) - { - // Call one dummy load of the first 4 bytes. - // This enforces loading all buffers and also detecting of the decryption key - SFileReadFile((HANDLE)hf, FileData, sizeof(FileData), &dwBytesRead, NULL); - pFileKeys[dwBlockIndex] = hf->dwFileKey; - FreeFileHandle(hf); - } - - // If we succeeded in reading 16 bytes from the file, - // we also know the encryption key - if(dwBytesRead == sizeof(FileData)) - continue; - } -*/ + // We don't know the encryption key of this file, // thus we cannot compact the file nError = ERROR_UNKNOWN_FILE_NAMES; @@ -471,11 +446,68 @@ static int CopyMpqFiles(TMPQArchive * ha, LPDWORD pFileKeys, TFileStream * pNewS return nError; } - /*****************************************************************************/ /* Public functions */ /*****************************************************************************/ +//----------------------------------------------------------------------------- +// Changing hash table size + +DWORD WINAPI SFileGetMaxFileCount(HANDLE hMpq) +{ + TMPQArchive * ha = (TMPQArchive *)hMpq; + + return ha->dwMaxFileCount; +} + +bool WINAPI SFileSetMaxFileCount(HANDLE hMpq, DWORD dwMaxFileCount) +{ + TMPQArchive * ha = (TMPQArchive *)hMpq; + DWORD dwNewHashTableSize = 0; + int nError = ERROR_SUCCESS; + + // Test the valid parameters + if(!IsValidMpqHandle(hMpq)) + nError = ERROR_INVALID_HANDLE; + if(ha->dwFlags & MPQ_FLAG_READ_ONLY) + nError = ERROR_ACCESS_DENIED; + if(dwMaxFileCount < ha->dwFileTableSize) + nError = ERROR_DISK_FULL; + + // ALL file names must be known in order to be able to rebuild hash table + if(nError == ERROR_SUCCESS && ha->pHashTable != NULL) + { + nError = CheckIfAllFilesKnown(ha); + if(nError == ERROR_SUCCESS) + { + // Calculate the hash table size for the new file limit + dwNewHashTableSize = GetHashTableSizeForFileCount(dwMaxFileCount); + + // Rebuild both file tables + nError = RebuildFileTable(ha, dwNewHashTableSize); + } + } + + // We always have to rebuild the (attributes) file due to file table change + if(nError == ERROR_SUCCESS) + { + // Invalidate (listfile) and (attributes) + InvalidateInternalFiles(ha); + + // Rebuild the HET table, if we have any + if(ha->pHetTable != NULL) + nError = RebuildHetTable(ha); + } + + // Return the error + if(nError != ERROR_SUCCESS) + SetLastError(nError); + return (nError == ERROR_SUCCESS); +} + +//----------------------------------------------------------------------------- +// Archive compacting + bool WINAPI SFileSetCompactCallback(HANDLE hMpq, SFILE_COMPACT_CALLBACK pfnCompactCB, void * pvUserData) { TMPQArchive * ha = (TMPQArchive *) hMpq; @@ -491,9 +523,6 @@ bool WINAPI SFileSetCompactCallback(HANDLE hMpq, SFILE_COMPACT_CALLBACK pfnCompa return true; } -//----------------------------------------------------------------------------- -// Archive compacting - bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bReserved */) { TFileStream * pTempStream = NULL; @@ -625,58 +654,3 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR SetLastError(nError); return (nError == ERROR_SUCCESS); } - -//----------------------------------------------------------------------------- -// Changing hash table size - -DWORD WINAPI SFileGetMaxFileCount(HANDLE hMpq) -{ - TMPQArchive * ha = (TMPQArchive *)hMpq; - - return ha->dwMaxFileCount; -} - -bool WINAPI SFileSetMaxFileCount(HANDLE hMpq, DWORD dwMaxFileCount) -{ - TMPQArchive * ha = (TMPQArchive *)hMpq; - DWORD dwNewHashTableSize = 0; - int nError = ERROR_SUCCESS; - - // Test the valid parameters - if(!IsValidMpqHandle(hMpq)) - nError = ERROR_INVALID_HANDLE; - if(ha->dwFlags & MPQ_FLAG_READ_ONLY) - nError = ERROR_ACCESS_DENIED; - if(dwMaxFileCount < ha->dwFileTableSize) - nError = ERROR_DISK_FULL; - - // ALL file names must be known in order to be able to rebuild hash table - if(nError == ERROR_SUCCESS && ha->pHashTable != NULL) - { - nError = CheckIfAllFilesKnown(ha); - if(nError == ERROR_SUCCESS) - { - // Calculate the hash table size for the new file limit - dwNewHashTableSize = GetHashTableSizeForFileCount(dwMaxFileCount); - - // Rebuild both file tables - nError = RebuildFileTable(ha, dwNewHashTableSize); - } - } - - // We always have to rebuild the (attributes) file due to file table change - if(nError == ERROR_SUCCESS) - { - // Invalidate (listfile) and (attributes) - InvalidateInternalFiles(ha); - - // Rebuild the HET table, if we have any - if(ha->pHetTable != NULL) - nError = RebuildHetTable(ha); - } - - // Return the error - if(nError != ERROR_SUCCESS) - SetLastError(nError); - return (nError == ERROR_SUCCESS); -} diff --git a/src/SFileListFile.cpp b/src/SFileListFile.cpp index 90f71fa..f0d2122 100644 --- a/src/SFileListFile.cpp +++ b/src/SFileListFile.cpp @@ -77,22 +77,22 @@ static TListFileCache * CreateListFileCache(HANDLE hListFile, const char * szWil memset(pCache, 0, sizeof(TListFileCache) + cchWildCard); // Shall we copy the mask? - if(cchWildCard != NULL) + if(cchWildCard != 0) { pCache->szWildCard = (char *)(pCache + 1); memcpy(pCache->szWildCard, szWildCard, cchWildCard); } // Fill-in the rest of the cache pointers - pCache->pBegin = (LPBYTE)(pCache + 1) + cchWildCard + 1; + pCache->pBegin = (LPBYTE)(pCache + 1) + cchWildCard; // Load the entire listfile to the cache SFileReadFile(hListFile, pCache->pBegin, dwFileSize, &dwBytesRead, NULL); - if(dwFileSize != 0) + if(dwBytesRead != 0) { // Allocate pointers pCache->pPos = pCache->pBegin; - pCache->pEnd = pCache->pBegin + dwFileSize; + pCache->pEnd = pCache->pBegin + dwBytesRead; } else { @@ -302,6 +302,11 @@ static LPBYTE CreateListFile(TMPQArchive * ha, DWORD * pcbListFile) assert((size_t)(szListLine - szListFile) == cbListFile); } } + else + { + szListFile = STORM_ALLOC(char, 1); + cbListFile = 0; + } // Free the sort table STORM_FREE(SortTable); @@ -379,9 +384,6 @@ int SListFileSaveToMpq(TMPQArchive * ha) pbListFile = CreateListFile(ha, &cbListFile); if(pbListFile != NULL) { - // We expect it to be nonzero size - assert(cbListFile != 0); - // Determine the real flags for (listfile) if(ha->dwFileFlags1 == MPQ_FILE_EXISTS) ha->dwFileFlags1 = GetDefaultSpecialFileFlags(cbListFile, ha->pHeader->wFormatVersion); diff --git a/src/SFileReadFile.cpp b/src/SFileReadFile.cpp index 3293cf6..d81b4de 100644 --- a/src/SFileReadFile.cpp +++ b/src/SFileReadFile.cpp @@ -166,6 +166,10 @@ static int ReadMpqSectors(TMPQFile * hf, LPBYTE pbBuffer, DWORD dwByteOffset, DW // Is the file compressed by Blizzard's multiple compression ? if(pFileEntry->dwFlags & MPQ_FILE_COMPRESS) { + // Remember the last used compression + hf->dwCompression0 = pbInSector[0]; + + // Decompress the data if(ha->pHeader->wFormatVersion >= MPQ_FORMAT_VERSION_2) nResult = SCompDecompress2(pbOutSector, &cbOutSector, pbInSector, cbInSector); else @@ -289,6 +293,10 @@ static int ReadMpqFileSingleUnit(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos // Is the file compressed by Blizzard's multiple compression ? if(pFileEntry->dwFlags & MPQ_FILE_COMPRESS) { + // Remember the last used compression + hf->dwCompression0 = pbRawData[0]; + + // Decompress the file if(ha->pHeader->wFormatVersion >= MPQ_FORMAT_VERSION_2) nResult = SCompDecompress2(hf->pbFileSector, &cbOutBuffer, pbRawData, cbInBuffer); else @@ -395,6 +403,7 @@ static int ReadMpkFileSingleUnit(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos { int cbOutBuffer = (int)hf->dwDataSize; + hf->dwCompression0 = pbRawData[0]; if(!SCompDecompressMpk(hf->pbFileSector, &cbOutBuffer, pbRawData, (int)pFileEntry->dwCmpSize)) nError = ERROR_FILE_CORRUPT; } @@ -688,6 +697,9 @@ bool WINAPI SFileReadFile(HANDLE hFile, void * pvBuffer, DWORD dwToRead, LPDWORD } } + // Clear the last used compression + hf->dwCompression0 = 0; + // If the file is local file, read the data directly from the stream if(hf->pStream != NULL) { diff --git a/src/StormCommon.h b/src/StormCommon.h index 3363a70..d4bf57c 100644 --- a/src/StormCommon.h +++ b/src/StormCommon.h @@ -178,6 +178,8 @@ ULONGLONG CalculateRawSectorOffset(TMPQFile * hf, DWORD dwSectorOffset); int ConvertMpqHeaderToFormat4(TMPQArchive * ha, ULONGLONG MpqOffset, ULONGLONG FileSize, DWORD dwFlags); +bool IsValidHashEntry(TMPQArchive * ha, TMPQHash * pHash); + TMPQHash * FindFreeHashEntry(TMPQArchive * ha, DWORD dwStartIndex, DWORD dwName1, DWORD dwName2, LCID lcLocale); TMPQHash * GetFirstHashEntry(TMPQArchive * ha, const char * szFileName); TMPQHash * GetNextHashEntry(TMPQArchive * ha, TMPQHash * pFirstHash, TMPQHash * pPrevHash); @@ -245,6 +247,7 @@ int SCompDecompressMpk(void * pvOutBuffer, int * pcbOutBuffer, void * pvInBuffer // Common functions - MPQ File TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry); +TMPQFile * CreateWritableHandle(TMPQArchive * ha, DWORD dwFileSize); void * LoadMpqTable(TMPQArchive * ha, ULONGLONG ByteOffset, DWORD dwCompressedSize, DWORD dwRealSize, DWORD dwKey, bool * pbTableIsCut); int AllocateSectorBuffer(TMPQFile * hf); int AllocatePatchInfo(TMPQFile * hf, bool bLoadFromFile); @@ -303,6 +306,12 @@ int SFileAddFile_Init( TMPQFile ** phf ); +int SFileAddFile_Init( + TMPQArchive * ha, + TMPQFile * hfSrc, + TMPQFile ** phf + ); + int SFileAddFile_Write( TMPQFile * hf, const void * pvData, @@ -339,12 +348,14 @@ int SSignFileFinish(TMPQArchive * ha); void DumpMpqHeader(TMPQHeader * pHeader); void DumpHashTable(TMPQHash * pHashTable, DWORD dwHashTableSize); void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable); +void DumpFileTable(TFileEntry * pFileTable, DWORD dwFileTableSize); #else -#define DumpMpqHeader(h) /* */ -#define DumpHashTable(h, s) /* */ -#define DumpHetAndBetTable(h, b) /* */ +#define DumpMpqHeader(h) /* */ +#define DumpHashTable(t, s) /* */ +#define DumpHetAndBetTable(t, s) /* */ +#define DumpFileTable(t, s) /* */ #endif diff --git a/test/StormTest.cpp b/test/StormTest.cpp index 06e599f..f9d707a 100644 --- a/test/StormTest.cpp +++ b/test/StormTest.cpp @@ -288,6 +288,30 @@ static bool IsMpqExtension(const char * szFileName) return false; } +static void AddStringBeforeExtension(char * szBuffer, const char * szFileName, const char * szExtraString) +{ + const char * szExtension; + size_t nLength; + + // Get the extension + szExtension = strrchr(szFileName, '.'); + if(szExtension == NULL) + szExtension = szFileName + strlen(szFileName); + nLength = (size_t)(szExtension - szFileName); + + // Copy the part before extension + memcpy(szBuffer, szFileName, nLength); + szFileName += nLength; + szBuffer += nLength; + + // Append the extra data + if(szExtraString != NULL) + strcpy(szBuffer, szExtraString); + + // Append the rest of the file name + strcat(szBuffer, szFileName); +} + static bool CompareBlocks(LPBYTE pbBlock1, LPBYTE pbBlock2, DWORD dwLength, DWORD * pdwDifference) { for(DWORD i = 0; i < dwLength; i++) @@ -2357,7 +2381,7 @@ static int TestOpenArchive(const char * szPlainName, const char * szListFile = N bool bIsPartialMpq = false; int nError; - // If the file is a partial MPQ, don;t load all files + // If the file is a partial MPQ, don't load all files bIsPartialMpq = (strstr(szPlainName, ".MPQ.part") != NULL); // Copy the archive so we won't fuck up the original one @@ -3069,7 +3093,77 @@ static int TestAddFile_ListFileTest(const char * szSourceMpq, bool bShouldHaveLi SFileCloseArchive(hMpq); return nError; } +/* +static int TestCreateArchive_Deprotect(const char * szPlainName) +{ + TLogHelper Logger("DeprotectTest", szPlainName); + HANDLE hMpq1 = NULL; + HANDLE hMpq2 = NULL; + char szMpqName1[MAX_PATH]; + char szMpqName2[MAX_PATH]; + BYTE FileHash1[MD5_DIGEST_SIZE]; + BYTE FileHash2[MD5_DIGEST_SIZE]; + DWORD dwFileCount1 = 0; + DWORD dwFileCount2 = 0; + DWORD dwTestFlags = TEST_FLAG_LOAD_FILES | TEST_FLAG_HASH_FILES; + int nError = ERROR_SUCCESS; + + // First copy: The original (untouched) file + if(nError == ERROR_SUCCESS) + { + AddStringBeforeExtension(szMpqName1, szPlainName, "_original"); + nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szMpqName1, &hMpq1); + if(nError != ERROR_SUCCESS) + Logger.PrintMessage("Failed to open %s", szMpqName1); + } + + // Second copy: Will be deprotected + if(nError == ERROR_SUCCESS) + { + AddStringBeforeExtension(szMpqName2, szPlainName, "_deprotected"); + nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szMpqName2, &hMpq2); + if(nError != ERROR_SUCCESS) + Logger.PrintMessage("Failed to open %s", szMpqName2); + } + // Deprotect the second map + if(nError == ERROR_SUCCESS) + { + SFileSetCompactCallback(hMpq2, CompactCallback, &Logger); + if(!SFileCompactArchive(hMpq2, NULL, false)) + nError = Logger.PrintError("Failed to deprotect archive %s", szMpqName2); + } + + // Calculate number of files and compare their hash (archive 1) + if(nError == ERROR_SUCCESS) + { + memset(FileHash1, 0, sizeof(FileHash1)); + nError = SearchArchive(&Logger, hMpq1, dwTestFlags, &dwFileCount1, FileHash1); + } + + // Calculate number of files and compare their hash (archive 2) + if(nError == ERROR_SUCCESS) + { + memset(FileHash1, 0, sizeof(FileHash2)); + nError = SearchArchive(&Logger, hMpq2, dwTestFlags, &dwFileCount2, FileHash2); + } + + if(nError == ERROR_SUCCESS) + { + if(dwFileCount1 != dwFileCount2) + Logger.PrintMessage("Different file count (%u in %s; %u in %s)", dwFileCount1, szMpqName1, dwFileCount2, szMpqName2); + if(memcmp(FileHash1, FileHash2, MD5_DIGEST_SIZE)) + Logger.PrintMessage("Different file hash (%s vs %s)", szMpqName1, szMpqName2); + } + + // Close both MPQs + if(hMpq2 != NULL) + SFileCloseArchive(hMpq2); + if(hMpq1 != NULL) + SFileCloseArchive(hMpq1); + return nError; +} +*/ static int TestCreateArchive_EmptyMpq(const char * szPlainName, DWORD dwCreateFlags) { TLogHelper Logger("CreateEmptyMpq", szPlainName); @@ -4029,7 +4123,7 @@ int main(int argc, char * argv[]) // Search in listfile if(nError == ERROR_SUCCESS) nError = TestSearchListFile("ListFile_Blizzard.txt"); -/* + // Test opening local file with SFileOpenFileEx if(nError == ERROR_SUCCESS) nError = TestOpenLocalFile("ListFile_Blizzard.txt"); @@ -4057,23 +4151,44 @@ int main(int argc, char * argv[]) // Open a truncated archive if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2002_v1_BlockTableCut.MPQ"); -*/ - // Open an Warcraft III map locked by a protector + + // Open a MPQ that actually has user data + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("MPQ_2010_v2_HasUserData.s2ma"); + + // Open an Warcraft III map whose "(attributes)" file has (BlockTableSize-1) entries + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("MPQ_2014_v1_AttributesOneEntryLess.w3x"); + + // Open a MPQ archive v 3.0 + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("MPQ_2010_v3_expansion-locale-frFR.MPQ"); + + // Open an encrypted archive from Starcraft II installer + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("mpqe-file://MPQ_2011_v2_EncryptedMpq.MPQE"); + + // Open a MPK archive from Longwu online + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("MPx_2013_v1_LongwuOnline.mpk"); + + // Open a SQP archive from War of the Immortals + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("MPx_2013_v1_WarOfTheImmortals.sqp", "ListFile_WarOfTheImmortals.txt"); + + // Open a partial MPQ with compressed hash table + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("part-file://MPQ_2010_v2_HashTableCompressed.MPQ.part"); + if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_HashTable_FakeValid.w3x"); -/* - // Open an Warcraft III map locked by a protector + if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_InvalidUserData.w3x"); - // Open an Warcraft III map locked by a protector if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_InvalidMpqFormat.w3x"); - // Open a MPQ that actually has user data - if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("MPQ_2010_v2_HasUserData.s2ma"); - // Open an Warcraft III map locked by the Spazzler protector if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_Spazzler.w3x"); @@ -4104,30 +4219,6 @@ int main(int argc, char * argv[]) if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2015_v1_flem1.w3x"); - // Open an Warcraft III map whose "(attributes)" file has (BlockTableSize-1) entries - if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("MPQ_2014_v1_AttributesOneEntryLess.w3x"); - - // Open a MPQ archive v 3.0 - if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("MPQ_2010_v3_expansion-locale-frFR.MPQ"); - - // Open an encrypted archive from Starcraft II installer - if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("mpqe-file://MPQ_2011_v2_EncryptedMpq.MPQE"); - - // Open a MPK archive from Longwu online - if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("MPx_2013_v1_LongwuOnline.mpk"); - - // Open a SQP archive from War of the Immortals - if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("MPx_2013_v1_WarOfTheImmortals.sqp", "ListFile_WarOfTheImmortals.txt"); - - // Open a partial MPQ with compressed hash table - if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("part-file://MPQ_2010_v2_HashTableCompressed.MPQ.part"); - // Open the multi-file archive with wrong prefix to see how StormLib deals with it if(nError == ERROR_SUCCESS) nError = TestOpenArchive_WillFail("flat-file://streaming/model.MPQ.0"); @@ -4312,7 +4403,7 @@ int main(int argc, char * argv[]) // Test replacing a file with zero size file if(nError == ERROR_SUCCESS) nError = TestModifyArchive_ReplaceFile("MPQ_2014_v4_Base.StormReplay", "AddFile-replay.message.events"); -*/ + #ifdef _MSC_VER _CrtDumpMemoryLeaks(); #endif // _MSC_VER |