diff options
| author | unknown <E:\Ladik\Mail> | 2015-05-01 07:06:29 +0200 | 
|---|---|---|
| committer | unknown <E:\Ladik\Mail> | 2015-05-01 07:06:29 +0200 | 
| commit | 46930855f500c1b494e3b16bb7a3323c07d4d5fb (patch) | |
| tree | 6220cc643761137a930841d8ec828db9f3db53cf | |
| parent | a205159d004871efbedd7cbfb686b8fe82bfb532 (diff) | |
+ Removed back reference of FileTable -> HashTable, as it is logically incorrect
+ Optimized patching process so it consimes less memory
+ Added hash table and block table defragmenting for malformed War3 maps
| -rw-r--r-- | src/SBaseCommon.cpp | 16 | ||||
| -rw-r--r-- | src/SBaseDumpData.cpp | 52 | ||||
| -rw-r--r-- | src/SBaseFileTable.cpp | 1032 | ||||
| -rw-r--r-- | src/SFileAddFile.cpp | 245 | ||||
| -rw-r--r-- | src/SFileAttributes.cpp | 63 | ||||
| -rw-r--r-- | src/SFileCompactArchive.cpp | 33 | ||||
| -rw-r--r-- | src/SFileCreateArchive.cpp | 12 | ||||
| -rw-r--r-- | src/SFileFindFile.cpp | 16 | ||||
| -rw-r--r-- | src/SFileGetFileInfo.cpp | 43 | ||||
| -rw-r--r-- | src/SFileListFile.cpp | 24 | ||||
| -rw-r--r-- | src/SFileOpenArchive.cpp | 61 | ||||
| -rw-r--r-- | src/SFileOpenFileEx.cpp | 316 | ||||
| -rw-r--r-- | src/SFilePatchArchives.cpp | 747 | ||||
| -rw-r--r-- | src/SFileReadFile.cpp | 30 | ||||
| -rw-r--r-- | src/SFileVerify.cpp | 11 | ||||
| -rw-r--r-- | src/StormCommon.h | 62 | ||||
| -rw-r--r-- | src/StormLib.h | 65 | ||||
| -rw-r--r-- | src/huffman/huff.cpp | 3 | ||||
| -rw-r--r-- | test/StormTest.cpp | 58 | 
19 files changed, 1361 insertions, 1528 deletions
| diff --git a/src/SBaseCommon.cpp b/src/SBaseCommon.cpp index 062a737..b4d62e0 100644 --- a/src/SBaseCommon.cpp +++ b/src/SBaseCommon.cpp @@ -97,7 +97,7 @@ unsigned char AsciiToUpperTable_Slash[256] =  #define STORM_BUFFER_SIZE       0x500 -#define HASH_INDEX_MASK(ha) (ha->dwHashTableSize ? (ha->dwHashTableSize - 1) : 0) +#define HASH_INDEX_MASK(ha) (ha->pHeader->dwHashTableSize ? (ha->pHeader->dwHashTableSize - 1) : 0)  static DWORD StormBuffer[STORM_BUFFER_SIZE];    // Buffer for the decryption engine  static bool  bMpqCryptographyInitialized = false; @@ -624,7 +624,8 @@ TMPQHash * GetNextHashEntry(TMPQArchive * ha, TMPQHash * pFirstHash, TMPQHash *  // Allocates an entry in the hash table  TMPQHash * AllocateHashEntry(      TMPQArchive * ha, -    TFileEntry * pFileEntry) +    TFileEntry * pFileEntry, +    LCID lcLocale)  {      TMPQHash * pHash;      DWORD dwStartIndex = ha->pfnHashString(pFileEntry->szFileName, MPQ_HASH_TABLE_INDEX); @@ -632,18 +633,15 @@ TMPQHash * AllocateHashEntry(      DWORD dwName2 = ha->pfnHashString(pFileEntry->szFileName, MPQ_HASH_NAME_B);      // Attempt to find a free hash entry -    pHash = FindFreeHashEntry(ha, dwStartIndex, dwName1, dwName2, pFileEntry->lcLocale); +    pHash = FindFreeHashEntry(ha, dwStartIndex, dwName1, dwName2, lcLocale);      if(pHash != NULL)      {          // Fill the free hash entry          pHash->dwName1      = dwName1;          pHash->dwName2      = dwName2; -        pHash->lcLocale     = pFileEntry->lcLocale; -        pHash->wPlatform    = pFileEntry->wPlatform; +        pHash->lcLocale     = (USHORT)lcLocale; +        pHash->wPlatform    = 0;          pHash->dwBlockIndex = (DWORD)(pFileEntry - ha->pFileTable); - -        // Fill the hash index in the file entry -        pFileEntry->dwHashIndex = (DWORD)(pHash - ha->pHashTable);      }      return pHash; @@ -1405,8 +1403,6 @@ void FreeFileHandle(TMPQFile *& hf)          // Then free all buffers allocated in the file structure          if(hf->pbFileData != NULL)              STORM_FREE(hf->pbFileData); -        if(hf->pPatchHeader != NULL) -            STORM_FREE(hf->pPatchHeader);          if(hf->pPatchInfo != NULL)              STORM_FREE(hf->pPatchInfo);          if(hf->SectorOffsets != NULL) diff --git a/src/SBaseDumpData.cpp b/src/SBaseDumpData.cpp index 7056d8b..d2b1206 100644 --- a/src/SBaseDumpData.cpp +++ b/src/SBaseDumpData.cpp @@ -41,6 +41,26 @@ void DumpMpqHeader(TMPQHeader * pHeader)      printf("-----------------------------------------------\n\n");  } +void DumpHashTable(TMPQHash * pHashTable, DWORD dwHashTableSize) +{ +    DWORD i; + +    if(pHashTable == NULL || dwHashTableSize == 0) +        return; + +    printf("== Hash Table =================================\n"); +    for(i = 0; i < dwHashTableSize; i++) +    { +        printf("[%08x] %08X %08X %04X %04X %08X\n", i, +                                                    pHashTable[i].dwName1, +                                                    pHashTable[i].dwName2, +                                                    pHashTable[i].lcLocale, +                                                    pHashTable[i].wPlatform, +                                                    pHashTable[i].dwBlockIndex); +    } +    printf("-----------------------------------------------\n\n"); +} +  void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable)  {      DWORD i; @@ -49,14 +69,14 @@ void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable)          return;      printf("== HET Header =================================\n"); -    printf("ULONGLONG  AndMask64         = %016llX\n", pHetTable->AndMask64);        -    printf("ULONGLONG  OrMask64          = %016llX\n", pHetTable->OrMask64);         +    printf("ULONGLONG  AndMask64         = %016llX\n",  pHetTable->AndMask64);        +    printf("ULONGLONG  OrMask64          = %016llX\n",  pHetTable->OrMask64);         +    printf("DWORD      dwEntryCount      = %08X\n",     pHetTable->dwEntryCount); +    printf("DWORD      dwTotalCount      = %08X\n",     pHetTable->dwTotalCount); +    printf("DWORD      dwNameHashBitSize = %08X\n",     pHetTable->dwNameHashBitSize);      printf("DWORD      dwIndexSizeTotal  = %08X\n",     pHetTable->dwIndexSizeTotal);      printf("DWORD      dwIndexSizeExtra  = %08X\n",     pHetTable->dwIndexSizeExtra);      printf("DWORD      dwIndexSize       = %08X\n",     pHetTable->dwIndexSize);      -    printf("DWORD      dwMaxFileCount    = %08X\n",     pHetTable->dwMaxFileCount);   -    printf("DWORD      dwHashTableSize   = %08X\n",     pHetTable->dwHashTableSize);  -    printf("DWORD      dwHashBitSize     = %08X\n",     pHetTable->dwHashBitSize);         printf("-----------------------------------------------\n\n");      printf("== BET Header =================================\n"); @@ -71,17 +91,17 @@ void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable)      printf("DWORD dwBitCount_CmpSize     = %08X\n",     pBetTable->dwBitCount_CmpSize);      printf("DWORD dwBitCount_FlagIndex   = %08X\n",     pBetTable->dwBitCount_FlagIndex);         printf("DWORD dwBitCount_Unknown     = %08X\n",     pBetTable->dwBitCount_Unknown); -    printf("DWORD dwBetHashSizeTotal     = %08X\n",     pBetTable->dwBetHashSizeTotal); -    printf("DWORD dwBetHashSizeExtra     = %08X\n",     pBetTable->dwBetHashSizeExtra); -    printf("DWORD dwBetHashSize          = %08X\n",     pBetTable->dwBetHashSize); -    printf("DWORD dwMaxFileCount         = %08X\n",     pBetTable->dwMaxFileCount); +    printf("DWORD dwBitTotal_NameHash2   = %08X\n",     pBetTable->dwBitTotal_NameHash2); +    printf("DWORD dwBitExtra_NameHash2   = %08X\n",     pBetTable->dwBitExtra_NameHash2); +    printf("DWORD dwBitCount_NameHash2   = %08X\n",     pBetTable->dwBitCount_NameHash2); +    printf("DWORD dwEntryCount           = %08X\n",     pBetTable->dwEntryCount);      printf("DWORD dwFlagCount            = %08X\n",     pBetTable->dwFlagCount);      printf("-----------------------------------------------\n\n");      printf("== HET & Bet Table ======================================================================\n\n");      printf("HetIdx HetHash BetIdx BetHash          ByteOffset       FileSize CmpSize  FlgIdx Flags   \n");      printf("------ ------- ------ ---------------- ---------------- -------- -------- ------ --------\n"); -    for(i = 0; i < pHetTable->dwHashTableSize; i++) +    for(i = 0; i < pHetTable->dwTotalCount; i++)      {          ULONGLONG ByteOffset = 0;          ULONGLONG BetHash = 0; @@ -96,14 +116,14 @@ void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable)                                         &dwBetIndex,                                          4); -        if(dwBetIndex < pHetTable->dwMaxFileCount) +        if(dwBetIndex < pHetTable->dwTotalCount)          {              DWORD dwEntryIndex = pBetTable->dwTableEntrySize * dwBetIndex; -            GetBits(pBetTable->pBetHashes, dwBetIndex * pBetTable->dwBetHashSizeTotal, -                                           pBetTable->dwBetHashSize, -                                          &BetHash, -                                           8); +            GetBits(pBetTable->pNameHashes, dwBetIndex * pBetTable->dwBitTotal_NameHash2, +                                            pBetTable->dwBitCount_NameHash2, +                                           &BetHash, +                                            8);              GetBits(pBetTable->pFileTable, dwEntryIndex + pBetTable->dwBitIndex_FilePos,                                             pBetTable->dwBitCount_FilePos, @@ -129,7 +149,7 @@ void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable)          }          printf(" %04X    %02lX     %04X  %016llX %016llX %08X %08X  %04X  %08X\n", i, -                                                         pHetTable->pHetHashes[i], +                                                         pHetTable->pNameHashes[i],                                                           dwBetIndex,                                                           BetHash,                                                           ByteOffset, diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp index 97ecb53..9e6d4b6 100644 --- a/src/SBaseFileTable.cpp +++ b/src/SBaseFileTable.cpp @@ -372,9 +372,9 @@ int ConvertMpqHeaderToFormat4(              //              Label_ArchiveVersion1: -            if(pHeader->dwHashTablePos <= pHeader->dwHeaderSize) +            if(pHeader->dwHashTablePos <= pHeader->dwHeaderSize || (pHeader->dwHashTablePos & 0x80000000))                  ha->dwFlags |= MPQ_FLAG_MALFORMED; -            if(pHeader->dwBlockTablePos <= pHeader->dwHeaderSize) +            if(pHeader->dwBlockTablePos <= pHeader->dwHeaderSize || (pHeader->dwBlockTablePos & 0x80000000))                  ha->dwFlags |= MPQ_FLAG_MALFORMED;              // Fill the rest of the header @@ -568,42 +568,35 @@ int ConvertMpqHeaderToFormat4(  //-----------------------------------------------------------------------------  // Support for hash table -static bool IsValidHashEntry(TMPQArchive * ha, TMPQHash * pHash) +// Hash entry verification when the file table does not exist yet +static bool IsValidHashEntry1(TMPQArchive * ha, TMPQHash * pHash, TMPQBlock * pBlockTable)  { -    // The block table index must be smaller than file table size -    // The block table entry's flags must have MPQ_FILE_EXISTS set -    return ( -            (pHash->dwBlockIndex < ha->dwFileTableSize) && -            (ha->pFileTable[pHash->dwBlockIndex].dwFlags & MPQ_FILE_EXISTS_MASK) == MPQ_FILE_EXISTS -           ); -} - -// Returns a hash table entry in the following order: -// 1) A hash table entry with the neutral locale -// 2) A hash table entry with any other locale -// 3) NULL -static TMPQHash * GetHashEntryAny(TMPQArchive * ha, const char * szFileName) -{ -    TMPQHash * pHashNeutral = NULL; -    TMPQHash * pFirstHash = GetFirstHashEntry(ha, szFileName); -    TMPQHash * pHashAny = NULL; -    TMPQHash * pHash = pFirstHash; +    ULONGLONG ByteOffset;     +    TMPQBlock * pBlock = pBlockTable + pHash->dwBlockIndex; -    // Parse the found hashes -    while(pHash != NULL) +    // Storm.dll does not perform this check. However, if there will +    // be an entry with (dwBlockIndex > dwBlockTableSize), the game would crash +    // Hence we assume that dwBlockIndex must be less than dwBlockTableSize +    if(pHash->dwBlockIndex < ha->pHeader->dwBlockTableSize)      { -        // If we found neutral hash, remember it -        if(pHash->lcLocale == 0) -            pHashNeutral = pHash; -        if(pHashAny == NULL) -            pHashAny = pHash; - -        // Get the next hash entry for that file -        pHash = GetNextHashEntry(ha, pFirstHash, pHash);  +        // Check whether this is an existing file +        // Also we do not allow to be file size greater than 2GB +        if((pBlock->dwFlags & MPQ_FILE_EXISTS) && (pBlock->dwFSize & 0x8000000) == 0) +        { +            // The begin of the file must be within the archive +            ByteOffset = FileOffsetFromMpqOffset(ha, pBlock->dwFilePos); +            return (ByteOffset < ha->FileSize); +        }      } -    // At the end, return neutral hash (if found), otherwise NULL -    return (pHashNeutral != NULL) ? pHashNeutral : pHashAny; +    return false; +} + +// Hash entry verification when the file table does not exist yet +static bool IsValidHashEntry2(TMPQArchive * ha, TMPQHash * pHash) +{ +    TFileEntry * pFileEntry = ha->pFileTable + pHash->dwBlockIndex; +    return ((pHash->dwBlockIndex < ha->dwFileTableSize) && (pFileEntry->dwFlags & MPQ_FILE_EXISTS)) ? true : false;  }  // Returns a hash table entry in the following order: @@ -620,7 +613,7 @@ static TMPQHash * GetHashEntryLocale(TMPQArchive * ha, const char * szFileName,      while(pHash != NULL)      {          // If the locales match, return it -        if(pHash->lcLocale == lcLocale) +        if(lcLocale == 0 && lcLocale == pHash->lcLocale)              return pHash;          // If we found neutral hash, remember it @@ -658,78 +651,172 @@ static TMPQHash * GetHashEntryExact(TMPQArchive * ha, const char * szFileName, L      return NULL;  } -static int BuildFileTableFromBlockTable( +// Defragment the file table so it does not contain any gaps +// Note: As long as all values of all TMPQHash::dwBlockIndex +// are not HASH_ENTRY_FREE, the startup search index does not matter. +// Hash table is circular, so as long as there is no terminator, +// all entries will be found. +static TMPQHash * DefragmentHashTable(      TMPQArchive * ha, -    TFileEntry * pFileTable, +    TMPQHash  * pHashTable,      TMPQBlock * pBlockTable)  { -    TFileEntry * pFileEntry;      TMPQHeader * pHeader = ha->pHeader; -    TMPQBlock * pBlock; -    TMPQHash * pHashEnd = ha->pHashTable + ha->dwHashTableSize; -    TMPQHash * pHash; +    TMPQHash * pHashTableEnd = pHashTable + pHeader->dwHashTableSize; +    TMPQHash * pSource = pHashTable; +    TMPQHash * pTarget = pHashTable; +    DWORD dwFirstFreeEntry; +    DWORD dwNewTableSize; -    // Parse the entire hash table -    for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) -    { -        // -        // Only if the block index is less than block table size. -        // An MPQ protector can place multiple invalid entries -        // into the hash table: -        // -        // a6d79af0 e61a0932 001e0000 0000770b <== Fake valid -        // a6d79af0 e61a0932 0000d761 0000dacb <== Fake valid -        // a6d79af0 e61a0932 00000000 0000002f <== Real file entry (block index is valid) -        // a6d79af0 e61a0932 00005a4f 000093bc <== Fake valid -        // +    // Sanity checks +    assert(pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1); +    assert(pHeader->HiBlockTablePos64 == 0); -        if(pHash->dwBlockIndex < pHeader->dwBlockTableSize) +    // Parse the hash table and move the entries to the begin of it +    for(pSource = pHashTable; pSource < pHashTableEnd; pSource++) +    { +        // Check whether this is a valid hash table entry +        if(IsValidHashEntry1(ha, pSource, pBlockTable))          { -            // Get the pointer to the file entry and the block entry -            pFileEntry = pFileTable + pHash->dwBlockIndex; -            pBlock = pBlockTable + pHash->dwBlockIndex; +            // Copy the hash table entry back +            if(pSource > pTarget) +                pTarget[0] = pSource[0]; -            // Only if the block table entry contains valid file -            if((pBlock->dwFlags & MPQ_FILE_EXISTS) && !(pBlock->dwFlags & ~MPQ_FILE_VALID_FLAGS)) -            { -                // ByteOffset is only valid if file size is not zero -                pFileEntry->ByteOffset = pBlock->dwFilePos; -                if(pFileEntry->ByteOffset == 0 && pBlock->dwFSize == 0) -                    pFileEntry->ByteOffset = ha->pHeader->dwHeaderSize; - -                pFileEntry->dwHashIndex = (DWORD)(pHash - ha->pHashTable); -                pFileEntry->dwFileSize  = pBlock->dwFSize; -                pFileEntry->dwCmpSize   = pBlock->dwCSize; -                pFileEntry->dwFlags     = pBlock->dwFlags; -                pFileEntry->lcLocale    = pHash->lcLocale; -                pFileEntry->wPlatform   = pHash->wPlatform; -            } +            // Move the target +            pTarget++;          }      } -    return ERROR_SUCCESS; +    // Calculate how many entries in the hash table we really need +    dwFirstFreeEntry = (DWORD)(pTarget - pHashTable); +    dwNewTableSize = GetHashTableSizeForFileCount(dwFirstFreeEntry); + +    // Fill the rest with entries that look like deleted +    pHashTableEnd = pHashTable + dwNewTableSize; +    pSource = pHashTable + dwFirstFreeEntry; +    memset(pSource, 0xFF, (dwNewTableSize - dwFirstFreeEntry) * sizeof(TMPQHash)); + +    // Mark the block indexes as deleted +    for(; pSource < pHashTableEnd; pSource++) +        pSource->dwBlockIndex = HASH_ENTRY_DELETED; + +    // Free some of the space occupied by the hash table +    if(dwNewTableSize < pHeader->dwHashTableSize) +    { +        pHashTable = STORM_REALLOC(TMPQHash, pHashTable, dwNewTableSize); +        ha->pHeader->dwHashTableSize = dwNewTableSize; +    } + +    return pHashTable;  } -static int UpdateFileTableFromHashTable( +static int BuildFileTableFromBlockTable(      TMPQArchive * ha, -    TFileEntry * pFileTable) +    TMPQBlock * pBlockTable)  {      TFileEntry * pFileEntry; -    TMPQHash * pHashEnd = ha->pHashTable + ha->dwHashTableSize; +    TMPQHeader * pHeader = ha->pHeader; +    TMPQBlock * pBlock; +    TMPQHash * pHashTableEnd;      TMPQHash * pHash; +    LPDWORD DefragmentTable = NULL; +    DWORD dwItemCount = 0; -    for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) +    // Sanity checks +    assert(ha->pFileTable != NULL); +    assert(ha->dwFileTableSize >= ha->dwMaxFileCount); + +    // Defragment the hash table, if needed +    if(ha->dwFlags & MPQ_FLAG_HASH_TABLE_CUT)      { -        if(pHash->dwBlockIndex < ha->dwFileTableSize) +        ha->pHashTable = DefragmentHashTable(ha, ha->pHashTable, pBlockTable); +        ha->dwMaxFileCount = pHeader->dwHashTableSize; +    } + +    // If the hash table or block table is cut, +    // we will defragment the block table +    if(ha->dwFlags & (MPQ_FLAG_HASH_TABLE_CUT | MPQ_FLAG_BLOCK_TABLE_CUT)) +    { +        // Sanity checks +        assert(pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1); +        assert(pHeader->HiBlockTablePos64 == 0); + +        // Allocate the translation table +        DefragmentTable = STORM_ALLOC(DWORD, pHeader->dwBlockTableSize); +        if(DefragmentTable == NULL) +            return ERROR_NOT_ENOUGH_MEMORY; + +        // Fill the translation table +        memset(DefragmentTable, 0xFF, pHeader->dwBlockTableSize * sizeof(DWORD)); +    } + +    // Parse the entire hash table +    pHashTableEnd = ha->pHashTable + pHeader->dwHashTableSize; +    for(pHash = ha->pHashTable; pHash < pHashTableEnd; pHash++) +    { +        DWORD dwBlockIndex = pHash->dwBlockIndex; +        DWORD dwNewIndex = pHash->dwBlockIndex; + +        // +        // We need to properly handle these cases: +        // - Multiple hash entries (same file name) point to the same block entry +        // - Multiple hash entries (different file name) point to the same block entry +        // +        // Ignore all hash table entries where: +        // - dwBlockIndex >= BlockTableSize +        // - Flags of the appropriate block table entry + +        if(IsValidHashEntry1(ha, pHash, pBlockTable))          { -            pFileEntry = pFileTable + pHash->dwBlockIndex; -            if(pFileEntry->dwFlags & MPQ_FILE_EXISTS) +            // Determine the new block index +            if(DefragmentTable != NULL)              { -                pFileEntry->dwHashIndex = (DWORD)(pHash - ha->pHashTable); -                pFileEntry->lcLocale    = pHash->lcLocale; -                pFileEntry->wPlatform   = pHash->wPlatform; +                // Need to handle case when multile hash +                // entries point to the same block entry +                if(DefragmentTable[dwBlockIndex] == HASH_ENTRY_FREE) +                { +                    DefragmentTable[dwBlockIndex] = dwItemCount; +                    dwNewIndex = dwItemCount++; +                } +                else +                { +                    dwNewIndex = DefragmentTable[dwBlockIndex]; +                } + +                // Fix the pointer in the hash entry +                pHash->dwBlockIndex = dwNewIndex;              } + +            // Get the pointer to the file entry and the block entry +            pFileEntry = ha->pFileTable + dwNewIndex; +            pBlock = pBlockTable + dwNewIndex; + +            // ByteOffset is only valid if file size is not zero +            pFileEntry->ByteOffset = pBlock->dwFilePos; +            if(pFileEntry->ByteOffset == 0 && pBlock->dwFSize == 0) +                pFileEntry->ByteOffset = ha->pHeader->dwHeaderSize; + +            // Fill the rest of the file entry +            pFileEntry->dwFileSize  = pBlock->dwFSize; +            pFileEntry->dwCmpSize   = pBlock->dwCSize; +            pFileEntry->dwFlags     = pBlock->dwFlags; +        } +    } + +    // Free the translation table +    if(DefragmentTable != NULL) +    { +        // If we defragmented the block table in the process, +        // free some memory by shrinking the file table +        if(ha->dwFileTableSize > ha->dwMaxFileCount) +        { +            ha->pFileTable = STORM_REALLOC(TFileEntry, ha->pFileTable, ha->dwMaxFileCount); +            ha->pHeader->dwBlockTableSize = ha->dwMaxFileCount; +            ha->dwFileTableSize = ha->dwMaxFileCount;          } + +        // Free the translation table +        STORM_FREE(DefragmentTable);      }      return ERROR_SUCCESS; @@ -743,11 +830,11 @@ static TMPQHash * TranslateHashTable(      size_t HashTableSize;      // Allocate copy of the hash table -    pHashTable = STORM_ALLOC(TMPQHash, ha->dwHashTableSize); +    pHashTable = STORM_ALLOC(TMPQHash, ha->pHeader->dwHashTableSize);      if(pHashTable != NULL)      {          // Copy the hash table -        HashTableSize = sizeof(TMPQHash) * ha->dwHashTableSize; +        HashTableSize = sizeof(TMPQHash) * ha->pHeader->dwHashTableSize;          memcpy(pHashTable, ha->pHashTable, HashTableSize);          // Give the size to the caller @@ -769,18 +856,17 @@ TMPQBlock * TranslateBlockTable(      TFileEntry * pFileEntry = ha->pFileTable;      TMPQBlock * pBlockTable;      TMPQBlock * pBlock; -    size_t NeedHiBlockTable = 0; -    size_t BlockTableSize; +    DWORD dwBlockTableSize = ha->pHeader->dwBlockTableSize; +    DWORD NeedHiBlockTable = 0;      // Allocate copy of the hash table -    pBlockTable = pBlock = STORM_ALLOC(TMPQBlock, ha->dwFileTableSize); +    pBlockTable = pBlock = STORM_ALLOC(TMPQBlock, dwBlockTableSize);      if(pBlockTable != NULL)      { -        // Copy the block table -        BlockTableSize = sizeof(TMPQBlock) * ha->dwFileTableSize; -        for(DWORD i = 0; i < ha->dwFileTableSize; i++) +        // Convert the block table +        for(DWORD i = 0; i < dwBlockTableSize; i++)          { -            NeedHiBlockTable |= (pFileEntry->ByteOffset >> 32); +            NeedHiBlockTable |= (DWORD)(pFileEntry->ByteOffset >> 32);              pBlock->dwFilePos = (DWORD)pFileEntry->ByteOffset;              pBlock->dwFSize   = pFileEntry->dwFileSize;              pBlock->dwCSize   = pFileEntry->dwCmpSize; @@ -792,7 +878,7 @@ TMPQBlock * TranslateBlockTable(          // Give the size to the caller          if(pcbTableSize != NULL) -            *pcbTableSize = (ULONGLONG)BlockTableSize; +            *pcbTableSize = (ULONGLONG)dwBlockTableSize * sizeof(TMPQBlock);          if(pbNeedHiBlockTable != NULL)              *pbNeedHiBlockTable = NeedHiBlockTable ? true : false; @@ -808,20 +894,19 @@ static USHORT * TranslateHiBlockTable(      TFileEntry * pFileEntry = ha->pFileTable;      USHORT * pHiBlockTable;      USHORT * pHiBlock; -    size_t HiBlockTableSize; +    DWORD dwBlockTableSize = ha->pHeader->dwBlockTableSize;      // Allocate copy of the hash table -    pHiBlockTable = pHiBlock = STORM_ALLOC(USHORT, ha->dwFileTableSize); +    pHiBlockTable = pHiBlock = STORM_ALLOC(USHORT, dwBlockTableSize);      if(pHiBlockTable != NULL)      {          // Copy the block table -        HiBlockTableSize = sizeof(USHORT) * ha->dwFileTableSize; -        for(DWORD i = 0; i < ha->dwFileTableSize; i++) +        for(DWORD i = 0; i < dwBlockTableSize; i++)              pHiBlock[i] = (USHORT)(pFileEntry[i].ByteOffset >> 0x20);          // Give the size to the caller          if(pcbTableSize != NULL) -            *pcbTableSize = (ULONGLONG)HiBlockTableSize; +            *pcbTableSize = (ULONGLONG)dwBlockTableSize * sizeof(USHORT);      }      return pHiBlockTable; @@ -1267,7 +1352,7 @@ static TMPQExtHeader * TranslateHetTable(TMPQHetTable * pHetTable, ULONGLONG * p      return &pHetHeader->ExtHdr;  } -DWORD GetFileIndex_Het(TMPQArchive * ha, const char * szFileName) +static DWORD GetFileIndex_Het(TMPQArchive * ha, const char * szFileName)  {      TMPQHetTable * pHetTable = ha->pHetTable;      ULONGLONG FileNameHash; @@ -1306,17 +1391,13 @@ DWORD GetFileIndex_Het(TMPQArchive * ha, const char * szFileName)              GetBits(pHetTable->pBetIndexes, pHetTable->dwIndexSizeTotal * Index,                                              pHetTable->dwIndexSize,                                             &dwFileIndex, -                                            4); -            // -            // TODO: This condition only happens when we are opening a MPQ -            // where some files were deleted by StormLib. Perhaps  -            // we should not allow shrinking of the file table in MPQs v 4.0? -            // assert(dwFileIndex <= ha->dwFileTableSize); -            // +                                            sizeof(DWORD));              // Verify the FileNameHash against the entry in the table of name hashes              if(dwFileIndex <= ha->dwFileTableSize && ha->pFileTable[dwFileIndex].FileNameHash == FileNameHash) +            {                  return dwFileIndex; +            }          }          // Move to the next entry in the HET table @@ -1368,6 +1449,7 @@ static void CreateBetHeader(      pBetHeader->ExtHdr.dwDataSize   = 0;      // Get the maximum values for the BET table +    pFileTableEnd = ha->pFileTable + ha->pHeader->dwBlockTableSize;      for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++)      {          // @@ -1416,7 +1498,7 @@ static void CreateBetHeader(                                         pBetHeader->dwBitCount_Unknown;      // Save the file count and flag count -    pBetHeader->dwEntryCount         = ha->dwFileTableSize; +    pBetHeader->dwEntryCount         = ha->pHeader->dwBlockTableSize;      pBetHeader->dwFlagCount          = dwMaxFlagIndex + 1;      pBetHeader->dwUnknown08          = 0x10; @@ -1577,6 +1659,7 @@ TMPQExtHeader * TranslateBetTable(              DWORD nBitOffset = 0;              // Construct the bit-based file table +            pFileTableEnd = ha->pFileTable + BetHeader.dwEntryCount;              for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++)              {                  // @@ -1678,11 +1761,25 @@ void FreeBetTable(TMPQBetTable * pBetTable)  //-----------------------------------------------------------------------------  // Support for file table -TFileEntry * GetFileEntryAny(TMPQArchive * ha, const char * szFileName) +TFileEntry * GetFileEntryLocale2(TMPQArchive * ha, const char * szFileName, LCID lcLocale, LPDWORD PtrHashIndex)  {      TMPQHash * pHash;      DWORD dwFileIndex; +    // First, we have to search the classic hash table +    // This is because on renaming, deleting, or changing locale, +    // we will need the pointer to hash table entry +    if(ha->pHashTable != NULL) +    { +        pHash = GetHashEntryLocale(ha, szFileName, lcLocale); +        if(pHash != NULL && pHash->dwBlockIndex < ha->dwFileTableSize) +        { +            if(PtrHashIndex != NULL) +                PtrHashIndex[0] = (DWORD)(pHash - ha->pHashTable); +            return ha->pFileTable + pHash->dwBlockIndex; +        } +    } +      // If we have HET table in the MPQ, try to find the file in HET table      if(ha->pHetTable != NULL)      { @@ -1691,76 +1788,48 @@ TFileEntry * GetFileEntryAny(TMPQArchive * ha, const char * szFileName)              return ha->pFileTable + dwFileIndex;      } -    // Otherwise, perform the file search in the classic hash table -    if(ha->pHashTable != NULL) -    { -        pHash = GetHashEntryAny(ha, szFileName); -        if(pHash != NULL && pHash->dwBlockIndex < ha->dwFileTableSize) -            return ha->pFileTable + pHash->dwBlockIndex; -    } -          // Not found      return NULL;  }  TFileEntry * GetFileEntryLocale(TMPQArchive * ha, const char * szFileName, LCID lcLocale)  { +    return GetFileEntryLocale2(ha, szFileName, lcLocale, NULL); +} + +TFileEntry * GetFileEntryExact(TMPQArchive * ha, const char * szFileName, LCID lcLocale, LPDWORD PtrHashIndex) +{      TMPQHash * pHash;      DWORD dwFileIndex; -    // If we have HET table in the MPQ, try to find the file in HET table -    if(ha->pHetTable != NULL) -    { -        dwFileIndex = GetFileIndex_Het(ha, szFileName); -        if(dwFileIndex != HASH_ENTRY_FREE) -            return ha->pFileTable + dwFileIndex; -    } - -    // Otherwise, perform the file search in the classic hash table +    // If the hash table is present, find the entry from hash table      if(ha->pHashTable != NULL)      { -        pHash = GetHashEntryLocale(ha, szFileName, lcLocale); +        pHash = GetHashEntryExact(ha, szFileName, lcLocale);          if(pHash != NULL && pHash->dwBlockIndex < ha->dwFileTableSize) +        { +            if(PtrHashIndex != NULL) +                PtrHashIndex[0] = (DWORD)(pHash - ha->pHashTable);              return ha->pFileTable + pHash->dwBlockIndex; +        }      } -     -    // Not found -    return NULL; -} - -TFileEntry * GetFileEntryExact(TMPQArchive * ha, const char * szFileName, LCID lcLocale) -{ -    TMPQHash * pHash; -    DWORD dwFileIndex;      // If we have HET table in the MPQ, try to find the file in HET table      if(ha->pHetTable != NULL)      {          dwFileIndex = GetFileIndex_Het(ha, szFileName);          if(dwFileIndex != HASH_ENTRY_FREE) +        { +            if(PtrHashIndex != NULL) +                PtrHashIndex[0] = HASH_ENTRY_FREE;              return ha->pFileTable + dwFileIndex; +        }      } - -    // Otherwise, perform the file search in the classic hash table -    if(ha->pHashTable != NULL) -    { -        pHash = GetHashEntryExact(ha, szFileName, lcLocale); -        if(pHash != NULL && pHash->dwBlockIndex < ha->dwFileTableSize) -            return ha->pFileTable + pHash->dwBlockIndex; -    } -     +         // Not found      return NULL;  } -TFileEntry * GetFileEntryByIndex(TMPQArchive * ha, DWORD dwIndex) -{ -    // For MPQs with classic hash table -    if(dwIndex < ha->dwFileTableSize) -        return ha->pFileTable + dwIndex; -    return NULL; -} -  void AllocateFileName(TMPQArchive * ha, TFileEntry * pFileEntry, const char * szFileName)  {      // Sanity check @@ -1792,99 +1861,103 @@ void AllocateFileName(TMPQArchive * ha, TFileEntry * pFileEntry, const char * sz      }  } -TFileEntry * AllocateFileEntry(TMPQArchive * ha, const char * szFileName, LCID lcLocale) +TFileEntry * AllocateFileEntry(TMPQArchive * ha, const char * szFileName, LCID lcLocale, LPDWORD PtrHashIndex)  { -    TFileEntry * pFileTableEnd; -    TFileEntry * pFileTablePtr; -    TFileEntry * pFileTable = ha->pFileTable; -    TFileEntry * pLastEntry = ha->pFileTable; -    TFileEntry * pFileEntry = NULL; -    TMPQHash * pHash; -    DWORD dwFreeNeeded = ha->dwReservedFiles; +    TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; +    TFileEntry * pFreeEntry = NULL; +    TFileEntry * pFileEntry; +    TMPQHash * pHash = NULL; +    DWORD dwReservedFiles = ha->dwReservedFiles;      DWORD dwFreeCount = 0; -    // The entry in the hash table should not be there. -    // This case must be handled by the caller -    assert(ha->pHashTable == NULL || GetHashEntryExact(ha, szFileName, lcLocale) == NULL); -    assert(ha->pHetTable == NULL || GetFileIndex_Het(ha, szFileName) == HASH_ENTRY_FREE); +    // Sanity check: File table size must be greater or equal to max file count +    assert(ha->dwFileTableSize >= ha->dwMaxFileCount); -    // Determine the end of the file table -    pFileTableEnd = ha->pFileTable + STORMLIB_MAX(ha->dwFileTableSize, ha->dwMaxFileCount); +    // If we are saving MPQ tables, we don't tale number of reserved files into account +    dwReservedFiles = (ha->dwFlags & MPQ_FLAG_SAVING_TABLES) ? 0 : ha->dwReservedFiles; -    // Find the first free file entry -    for(pFileTablePtr = pFileTable; pFileTablePtr < pFileTableEnd; pFileTablePtr++) +    // Now find a free entry in the file table. +    // Note that in the case when free entries are in the middle, +    // we need to use these +    for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++)      { -        // Is that entry free? -        // Note: Files with "delete marker" are not deleted. -        // Don't treat them as available -        if((pFileTablePtr->dwFlags & MPQ_FILE_EXISTS) == 0) +        if((pFileEntry->dwFlags & MPQ_FILE_EXISTS) == 0)          { -            if(pFileEntry == NULL) -                pFileEntry = pFileTablePtr; +            // Remember the first free entry +            if(pFreeEntry == NULL) +                pFreeEntry = pFileEntry;              dwFreeCount++; -        } -        else -        { -            // Update the last used entry -            if(pFileTablePtr > pLastEntry) -                pLastEntry = pFileTablePtr; +             +            // If the number of free items is greater than number +            // of reserved items, We can add the file +            if(dwFreeCount > dwReservedFiles) +                break;          }      } -    // Did we find an usable file entry? -    if(pFileEntry != NULL) +    // If the total number of free entries is less than number of reserved files, +    // we cannot add the file to the archive +    if(pFreeEntry == NULL || dwFreeCount <= dwReservedFiles) +        return NULL; + +    // Initialize the file entry and set its file name +    memset(pFreeEntry, 0, sizeof(TFileEntry)); +    AllocateFileName(ha, pFreeEntry, szFileName); + +    // If the archive has a hash table, we need to first free entry there +    if(ha->pHashTable != NULL)      { -        // Is there enough free entries? -        if(!(ha->dwFlags & MPQ_FLAG_SAVING_TABLES)) -            dwFreeNeeded++; -        if(dwFreeCount < dwFreeNeeded) -            return NULL; +        // Make sure that the entry is not there yet +        assert(GetHashEntryExact(ha, szFileName, lcLocale) == NULL); -        // Make sure that the entry is properly initialized -        memset(pFileEntry, 0, sizeof(TFileEntry)); -        pFileEntry->lcLocale = (USHORT)lcLocale; -        AllocateFileName(ha, pFileEntry, szFileName); +        // Find a free hash table entry for the name +        pHash = AllocateHashEntry(ha, pFreeEntry, lcLocale); +        if(pHash == NULL) +            return NULL; -        // If the MPQ has hash table, we have to insert the new entry into the hash table -        // We expect it to succeed because there must be a free hash entry if there is a free file entry -        // Note: Don't bother with the HET table. It will be rebuilt anyway -        if(ha->pHashTable != NULL) -        { -            pHash = AllocateHashEntry(ha, pFileEntry); -            assert(pHash != NULL); -        } +        // Set the file index to the hash table +        pHash->dwBlockIndex = (DWORD)(pFreeEntry - ha->pFileTable); +        PtrHashIndex[0] = (DWORD)(pHash - ha->pHashTable); +    } -        // Update the file table size -        if(pFileEntry >= pLastEntry) -            pLastEntry = pFileEntry; -        ha->dwFileTableSize = (DWORD)(pLastEntry - pFileTable) + 1; +    // If the archive has a HET table, just do some checks +    // Note: Don't bother modifying the HET table. It will be rebuilt from scratch after, anyway +    if(ha->pHetTable != NULL) +    { +        assert(GetFileIndex_Het(ha, szFileName) == HASH_ENTRY_FREE);      } -    // Return the file entry -    return pFileEntry; +    // Return the free table entry +    return pFreeEntry;  }  int RenameFileEntry(      TMPQArchive * ha, -    TFileEntry * pFileEntry, +    TMPQFile * hf,      const char * szNewFileName)  { -    TMPQHash * pHash; +    TFileEntry * pFileEntry = hf->pFileEntry; +    TMPQHash * pHashEntry = hf->pHashEntry; +    LCID lcLocale = 0; -    // Mark the entry as deleted in the hash table +    // If the archive hash hash table, we need to free the hash table entry      if(ha->pHashTable != NULL)      { -        assert(pFileEntry->dwHashIndex < ha->dwHashTableSize); +        // The file must have hash table entry assigned +        // Will exit if there are multiple HASH entries pointing to the same file entry +        if(pHashEntry == NULL) +            return ERROR_NOT_SUPPORTED; -        pHash = ha->pHashTable + pFileEntry->dwHashIndex; -        memset(pHash, 0xFF, sizeof(TMPQHash)); -        pHash->dwBlockIndex = HASH_ENTRY_DELETED; -    } +        // Save the locale +        lcLocale = pHashEntry->lcLocale; -    // -    // Note: Don't bother with the HET table. -    // It will be rebuilt from scratch anyway -    // +        // Mark the hash table entry as deleted +        pHashEntry->dwName1      = 0xFFFFFFFF; +        pHashEntry->dwName2      = 0xFFFFFFFF; +        pHashEntry->lcLocale     = 0xFFFF; +        pHashEntry->wPlatform    = 0xFFFF; +        pHashEntry->dwBlockIndex = HASH_ENTRY_DELETED; +    }      // Free the old file name      if(pFileEntry->szFileName != NULL) @@ -1894,44 +1967,36 @@ int RenameFileEntry(      // Allocate new file name      AllocateFileName(ha, pFileEntry, szNewFileName); -    // Now find a hash entry for the new file name +    // Allocate new hash entry      if(ha->pHashTable != NULL)      { -        // Try to find the hash table entry for the new file name -        // Note: Since we deleted one hash entry, this will always succeed -        pHash = AllocateHashEntry(ha, pFileEntry); -        assert(pHash != NULL); +        // Since we freed one hash entry before, this must succeed +        hf->pHashEntry = AllocateHashEntry(ha, pFileEntry, lcLocale); +        assert(hf->pHashEntry != NULL);      } -    // Invalidate the entries for (listfile) and (attributes) -    // After we are done with MPQ changes, we need to re-create them -    InvalidateInternalFiles(ha);      return ERROR_SUCCESS;  } -void DeleteFileEntry( -    TMPQArchive * ha, -    TFileEntry * pFileEntry) +int DeleteFileEntry(TMPQArchive * ha, TMPQFile * hf)  { -    TMPQHash * pHash; +    TFileEntry * pFileEntry = hf->pFileEntry; +    TMPQHash * pHashEntry = hf->pHashEntry; -    // If the MPQ has classic hash table, clear the entry there +    // If the archive hash hash table, we need to free the hash table entry      if(ha->pHashTable != NULL)      { -        // Only if the file entry is still an existing one -        if(pFileEntry->dwFlags & MPQ_FILE_EXISTS) -        { -            // We expect dwHashIndex to be within the hash table -            pHash = ha->pHashTable + pFileEntry->dwHashIndex; -            assert(pFileEntry->dwHashIndex < ha->dwHashTableSize); - -            // Set the hash table entry as deleted -            pHash->dwName1      = 0xFFFFFFFF; -            pHash->dwName2      = 0xFFFFFFFF; -            pHash->lcLocale     = 0xFFFF; -            pHash->wPlatform    = 0xFFFF; -            pHash->dwBlockIndex = HASH_ENTRY_DELETED; -        } +        // The file must have hash table entry assigned +        // Will exit if there are multiple HASH entries pointing to the same file entry +        if(pHashEntry == NULL) +            return ERROR_NOT_SUPPORTED; + +        // Mark the hash table entry as deleted +        pHashEntry->dwName1      = 0xFFFFFFFF; +        pHashEntry->dwName2      = 0xFFFFFFFF; +        pHashEntry->lcLocale     = 0xFFFF; +        pHashEntry->wPlatform    = 0xFFFF; +        pHashEntry->dwBlockIndex = HASH_ENTRY_DELETED;      }      // Free the file name, and set the file entry as deleted @@ -1940,67 +2005,73 @@ void DeleteFileEntry(      pFileEntry->szFileName = NULL;      // -    // Don't modify the HET table, because it gets recreated from scratch at every modification operation +    // Don't modify the HET table, because it gets recreated by the caller      // Don't decrement the number of entries in the file table -    // Keep Byte Offset, file size and compressed size in the file table -    // Also keep the CRC32 and MD5 in the file attributes -    // Clear the file name hash -    // Clear the MPQ_FILE_EXISTS bit. +    // Keep Byte Offset, file size, compressed size, CRC32 and MD5 +    // Clear the file name hash and the MPQ_FILE_EXISTS bit      // -    pFileEntry->FileNameHash = 0;      pFileEntry->dwFlags &= ~MPQ_FILE_EXISTS; +    pFileEntry->FileNameHash = 0; +    return ERROR_SUCCESS;      } -void InvalidateInternalFiles(TMPQArchive * ha) +DWORD InvalidateInternalFile(TMPQArchive * ha, const char * szFileName, DWORD dwFlagNone, DWORD dwFlagNew)  { -    TFileEntry * pFileEntry1 = NULL; -    TFileEntry * pFileEntry2 = NULL; -    TFileEntry * pFileEntry3 = NULL; +    TMPQFile * hf = NULL; +    DWORD dwFileFlags = 0; +    int nError = ERROR_FILE_NOT_FOUND; + +    // Open the file from the MPQ +    if(SFileOpenFileEx((HANDLE)ha, szFileName, SFILE_OPEN_BASE_FILE, (HANDLE *)&hf)) +    { +        // Remember the file flags +        dwFileFlags = hf->pFileEntry->dwFlags; + +        // Delete the file entry +        nError = DeleteFileEntry(ha, hf); +        if(nError == ERROR_SUCCESS) +        { +            ha->dwFlags |= dwFlagNew; +            ha->dwReservedFiles++; +        } +        // Free the file entry +        FreeFileHandle(hf); +    } + +    // If the deletion failed, set the "none" flag +    ha->dwFlags |= (nError != ERROR_SUCCESS) ? dwFlagNone : 0; +    return dwFileFlags; +} + +void InvalidateInternalFiles(TMPQArchive * ha) +{      // Do nothing if we are in the middle of saving internal files      if(!(ha->dwFlags & MPQ_FLAG_SAVING_TABLES))      {          // -        // We clear the file entries of (listfile) and (attributes) +        // We clear the file entries for (listfile), (attributes) and (signature)          // For each internal file cleared, we increment the number           // of reserved entries in the file table.          //          // Invalidate the (listfile), if not done yet -        if((ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) == 0) +        if((ha->dwFlags & (MPQ_FLAG_LISTFILE_NONE | MPQ_FLAG_LISTFILE_NEW)) == 0)          { -            // If the (listfile) is not there but will be, reserve one entry -            pFileEntry1 = GetFileEntryExact(ha, LISTFILE_NAME, LANG_NEUTRAL); -            if(pFileEntry1 == NULL && ha->dwFileFlags1) -                ha->dwReservedFiles++; -             -            // Mark the (listfile) as invalid -            ha->dwFlags |= MPQ_FLAG_LISTFILE_INVALID; +            ha->dwFileFlags1 = InvalidateInternalFile(ha, LISTFILE_NAME, MPQ_FLAG_LISTFILE_NONE, MPQ_FLAG_LISTFILE_NEW);          }          // Invalidate the (attributes), if not done yet -        if((ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) == 0) +        if((ha->dwFlags & (MPQ_FLAG_ATTRIBUTES_NONE | MPQ_FLAG_ATTRIBUTES_NEW)) == 0)          { -            // If the (attributes) is not there but will be, reserve one entry -            pFileEntry2 = GetFileEntryExact(ha, ATTRIBUTES_NAME, LANG_NEUTRAL); -            if(pFileEntry2 == NULL && ha->dwFileFlags2) -                ha->dwReservedFiles++; - -            // Mark (attributes) as invalid -            ha->dwFlags |= MPQ_FLAG_ATTRIBUTES_INVALID; +            ha->dwFileFlags2 = InvalidateInternalFile(ha, ATTRIBUTES_NAME, MPQ_FLAG_ATTRIBUTES_NONE, MPQ_FLAG_ATTRIBUTES_NEW);          }          // Invalidate the (signature), if not done yet -        if((ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) == 0) +        if((ha->dwFlags & (MPQ_FLAG_SIGNATURE_NONE | MPQ_FLAG_SIGNATURE_NEW)) == 0)          { -            // If the (signature) is not there but will be, reserve one entry -            pFileEntry3 = GetFileEntryExact(ha, SIGNATURE_NAME, LANG_NEUTRAL); -            if(pFileEntry3 == NULL && ha->dwFileFlags3) -                ha->dwReservedFiles++; - -            // Mark (signature) as invalid -            ha->dwFlags |= MPQ_FLAG_SIGNATURE_INVALID; +            ha->dwFileFlags3 = InvalidateInternalFile(ha, SIGNATURE_NAME, MPQ_FLAG_SIGNATURE_NONE, MPQ_FLAG_SIGNATURE_NEW);          }          // Remember that the MPQ has been changed @@ -2030,7 +2101,7 @@ int CreateHashTable(TMPQArchive * ha, DWORD dwHashTableSize)      // Fill it      memset(pHashTable, 0xFF, dwHashTableSize * sizeof(TMPQHash)); -    ha->dwHashTableSize = dwHashTableSize; +    ha->pHeader->dwHashTableSize = dwHashTableSize;      ha->dwMaxFileCount = dwHashTableSize;      ha->pHashTable = pHashTable;      return ERROR_SUCCESS; @@ -2065,8 +2136,9 @@ static TMPQHash * LoadHashTable(TMPQArchive * ha)              // Read, decrypt and uncompress the hash table              pHashTable = (TMPQHash *)LoadMpqTable(ha, ByteOffset, dwCmpSize, dwTableSize, MPQ_KEY_HASH_TABLE, &bHashTableIsCut); +//          DumpHashTable(pHashTable, pHeader->dwHashTableSize); -            // If the hash table was cut, we need to remember it +            // If the hash table was cut, we can/have to defragment it              if(pHashTable != NULL && bHashTableIsCut)                  ha->dwFlags |= (MPQ_FLAG_MALFORMED | MPQ_FLAG_HASH_TABLE_CUT);              break; @@ -2084,6 +2156,17 @@ static TMPQHash * LoadHashTable(TMPQArchive * ha)      return pHashTable;  } +int CreateFileTable(TMPQArchive * ha, DWORD dwFileTableSize) +{ +    ha->pFileTable = STORM_ALLOC(TFileEntry, dwFileTableSize); +    if(ha->pFileTable == NULL) +        return ERROR_NOT_ENOUGH_MEMORY; + +    memset(ha->pFileTable, 0x00, sizeof(TFileEntry) * dwFileTableSize); +    ha->dwFileTableSize = dwFileTableSize; +    return ERROR_SUCCESS; +} +  TMPQBlock * LoadBlockTable(TMPQArchive * ha, bool /* bDontFixEntries */)  {      TMPQHeader * pHeader = ha->pHeader; @@ -2202,42 +2285,30 @@ int LoadAnyHashTable(TMPQArchive * ha)      // Note: We don't care about HET table limits, because HET table is rebuilt      // after each file add/rename/delete.      ha->dwMaxFileCount = (ha->pHashTable != NULL) ? pHeader->dwHashTableSize : HASH_TABLE_SIZE_MAX; -    ha->dwHashTableSize = pHeader->dwHashTableSize;      return ERROR_SUCCESS;  } -static int BuildFileTable_Classic( -    TMPQArchive * ha, -    TFileEntry * pFileTable) +static int BuildFileTable_Classic(TMPQArchive * ha)  { -    TFileEntry * pFileEntry;      TMPQHeader * pHeader = ha->pHeader;      TMPQBlock * pBlockTable;      int nError = ERROR_SUCCESS;      // Sanity checks      assert(ha->pHashTable != NULL); +    assert(ha->pFileTable != NULL);      // If the MPQ has no block table, do nothing -    if(ha->pHeader->dwBlockTableSize == 0) +    if(pHeader->dwBlockTableSize == 0)          return ERROR_SUCCESS; +    assert(ha->dwFileTableSize >= pHeader->dwBlockTableSize);      // Load the block table +    // WARNING! ha->pFileTable can change in the process!!      pBlockTable = (TMPQBlock *)LoadBlockTable(ha);      if(pBlockTable != NULL)      { -        // If we don't have HET table, we build the file entries from the hash&block tables -        // If we do, we only update it from the hash table -        if(ha->pHetTable == NULL) -        { -            nError = BuildFileTableFromBlockTable(ha, pFileTable, pBlockTable); -        } -        else -        { -            nError = UpdateFileTableFromHashTable(ha, pFileTable); -        } - -        // Free the block table +        nError = BuildFileTableFromBlockTable(ha, pBlockTable);          STORM_FREE(pBlockTable);      }      else @@ -2265,15 +2336,14 @@ static int BuildFileTable_Classic(              // Now merge the hi-block table to the file table              if(nError == ERROR_SUCCESS)              { +                TFileEntry * pFileEntry = ha->pFileTable; + +                // Swap the hi-block table                  BSWAP_ARRAY16_UNSIGNED(pHiBlockTable, dwTableSize); -                pFileEntry = pFileTable;                  // Add the high file offset to the base file offset. -                for(DWORD i = 0; i < pHeader->dwBlockTableSize; i++) -                { +                for(DWORD i = 0; i < pHeader->dwBlockTableSize; i++, pFileEntry++)                      pFileEntry->ByteOffset = MAKE_OFFSET64(pHiBlockTable[i], pFileEntry->ByteOffset); -                    pFileEntry++; -                }              }              // Free the hi-block table @@ -2285,18 +2355,14 @@ static int BuildFileTable_Classic(          }      } -    // Set the current size of the file table -    ha->dwFileTableSize = pHeader->dwBlockTableSize;      return nError;  } -static int BuildFileTable_HetBet( -    TMPQArchive * ha, -    TFileEntry * pFileTable) +static int BuildFileTable_HetBet(TMPQArchive * ha)  {      TMPQHetTable * pHetTable = ha->pHetTable;      TMPQBetTable * pBetTable; -    TFileEntry * pFileEntry = pFileTable; +    TFileEntry * pFileEntry = ha->pFileTable;      TBitArray * pBitArray;      DWORD dwBitPosition = 0;      DWORD i; @@ -2317,7 +2383,7 @@ static int BuildFileTable_HetBet(              DWORD dwFileIndex = 0;              // Is the entry in the HET table occupied? -            if(pHetTable->pNameHashes[i] != 0) +            if(pHetTable->pNameHashes[i] != HET_ENTRY_FREE)              {                  // Load the index to the BET table                  GetBits(pHetTable->pBetIndexes, pHetTable->dwIndexSizeTotal * i, @@ -2337,14 +2403,14 @@ static int BuildFileTable_HetBet(                                                      8);                      // Combine both part of the name hash and put it to the file table -                    pFileEntry = pFileTable + dwFileIndex; +                    pFileEntry = ha->pFileTable + dwFileIndex;                      pFileEntry->FileNameHash = (NameHash1 << pBetTable->dwBitCount_NameHash2) | NameHash2;                  }              }          }          // Go through the entire BET table and convert it to the file table. -        pFileEntry = pFileTable; +        pFileEntry = ha->pFileTable;          pBitArray = pBetTable->pFileTable;           for(i = 0; i < pBetTable->dwEntryCount; i++)          { @@ -2389,7 +2455,6 @@ static int BuildFileTable_HetBet(          }          // Set the current size of the file table -        ha->dwFileTableSize = pBetTable->dwEntryCount;          FreeBetTable(pBetTable);          nError = ERROR_SUCCESS;      } @@ -2403,11 +2468,11 @@ static int BuildFileTable_HetBet(  int BuildFileTable(TMPQArchive * ha)  { -    TFileEntry * pFileTable;      DWORD dwFileTableSize;      bool bFileTableCreated = false;      // Sanity checks +    assert(ha->pFileTable == NULL);      assert(ha->dwFileTableSize == 0);      assert(ha->dwMaxFileCount != 0); @@ -2415,18 +2480,19 @@ int BuildFileTable(TMPQArchive * ha)      dwFileTableSize = STORMLIB_MAX(ha->pHeader->dwBlockTableSize, ha->dwMaxFileCount);      // Allocate the file table with size determined before -    pFileTable = STORM_ALLOC(TFileEntry, dwFileTableSize); -    if(pFileTable == NULL) +    ha->pFileTable = STORM_ALLOC(TFileEntry, dwFileTableSize); +    if(ha->pFileTable == NULL)          return ERROR_NOT_ENOUGH_MEMORY;      // Fill the table with zeros -    memset(pFileTable, 0, dwFileTableSize * sizeof(TFileEntry)); +    memset(ha->pFileTable, 0, dwFileTableSize * sizeof(TFileEntry)); +    ha->dwFileTableSize = dwFileTableSize;      // If we have HET table, we load file table from the BET table      // Note: If BET table is corrupt or missing, we set the archive as read only      if(ha->pHetTable != NULL)      { -        if(BuildFileTable_HetBet(ha, pFileTable) != ERROR_SUCCESS) +        if(BuildFileTable_HetBet(ha) != ERROR_SUCCESS)              ha->dwFlags |= MPQ_FLAG_READ_ONLY;          else              bFileTableCreated = true; @@ -2436,209 +2502,105 @@ int BuildFileTable(TMPQArchive * ha)      // Note: If block table is corrupt or missing, we set the archive as read only      if(ha->pHashTable != NULL)      { -        if(BuildFileTable_Classic(ha, pFileTable) != ERROR_SUCCESS) +        if(BuildFileTable_Classic(ha) != ERROR_SUCCESS)              ha->dwFlags |= MPQ_FLAG_READ_ONLY;          else              bFileTableCreated = true;      } -     -    // If something failed, we free the file table entry -    if(bFileTableCreated == false) + +    // Return result +    return bFileTableCreated ? ERROR_SUCCESS : ERROR_FILE_CORRUPT; +} + +/* +void UpdateBlockTableSize(TMPQArchive * ha) +{ +    TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; +    TFileEntry * pFileEntry; +    DWORD dwBlockTableSize = 0; + +    // Calculate the number of files +    for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++)      { -        STORM_FREE(pFileTable); -        return ERROR_FILE_CORRUPT; +        // If the source table entry is valid,  +        if(pFileEntry->dwFlags & MPQ_FILE_EXISTS) +            dwBlockTableSize = (DWORD)(pFileEntry - ha->pFileTable) + 1;      } -    // Assign it to the archive structure -    ha->pFileTable = pFileTable; -    return ERROR_SUCCESS; +    // Save the block table size to the MPQ header +    ha->pHeader->dwBlockTableSize = ha->dwReservedFiles + dwBlockTableSize;  } +*/  // Defragment the file table so it does not contain any gaps  int DefragmentFileTable(TMPQArchive * ha)  { -    TFileEntry * pNewTable; -    TMPQHash * pHashEnd = ha->pHashTable + ha->dwHashTableSize; -    TMPQHash * pHash; -    PDWORD NewIndexes; -    DWORD dwNewTableSize = 0; -    DWORD dwNewBlockIndex; +    TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; +    TFileEntry * pSource = ha->pFileTable; +    TFileEntry * pTarget = ha->pFileTable; +    LPDWORD DefragmentTable; +    DWORD dwBlockTableSize = 0; +    DWORD dwSrcIndex; +    DWORD dwTrgIndex;      // Allocate brand new file table -    NewIndexes = STORM_ALLOC(DWORD, ha->dwFileTableSize); -    if(NewIndexes != NULL) +    DefragmentTable = STORM_ALLOC(DWORD, ha->dwFileTableSize); +    if(DefragmentTable != NULL)      {          // Clear the file table -        memset(NewIndexes, 0xFF, sizeof(DWORD) * ha->dwFileTableSize); +        memset(DefragmentTable, 0xFF, sizeof(DWORD) * ha->dwFileTableSize); -        // Create map of OldBlockIndex => NewBlockIndex -        for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) +        // Parse the entire file table and defragment it +        for(; pSource < pFileTableEnd; pSource++)          { -            // If the entry is valid, move it -            if(IsValidHashEntry(ha, pHash)) +            // If the source table entry is valid,  +            if(pSource->dwFlags & MPQ_FILE_EXISTS)              { -                // Was the new index already resolved? -                // Note: Multiple hash table entries can point to the same block table entry -                if(NewIndexes[pHash->dwBlockIndex] == 0xFFFFFFFF) -                    NewIndexes[pHash->dwBlockIndex] = dwNewTableSize++; +                // Remember the index conversion +                dwSrcIndex = (DWORD)(pSource - ha->pFileTable); +                dwTrgIndex = (DWORD)(pTarget - ha->pFileTable); +                DefragmentTable[dwSrcIndex] = dwTrgIndex; + +                // Move the entry, if needed +                if(pTarget != pSource) +                    pTarget[0] = pSource[0]; +                pTarget++; + +                // Update the block table size +                dwBlockTableSize = (DWORD)(pSource - ha->pFileTable) + 1;              }          } -        // We have to append reserved files -        dwNewTableSize += ha->dwReservedFiles; - -        // Allocate new file table -        pNewTable = STORM_ALLOC(TFileEntry, ha->dwMaxFileCount); -        if(pNewTable != NULL) +        // Did we defragment something? +        if(pTarget < pFileTableEnd)          { -            // Fill the new table with zeros -            memset(pNewTable, 0, sizeof(TFileEntry) * ha->dwMaxFileCount); +            // Clear the remaining file entries +            memset(pTarget, 0, (pFileTableEnd - pTarget) * sizeof(TFileEntry)); -            // Create map of OldBlockIndex => NewBlockIndex -            for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) +            // Go through the hash table and relocate the block indexes +            if(ha->pHashTable != NULL)              { -                // If the entry is valid, move it -                if(IsValidHashEntry(ha, pHash)) -                { -                    // The new index should be resolved by now -                    assert(NewIndexes[pHash->dwBlockIndex] != 0xFFFFFFFF); -                    dwNewBlockIndex = NewIndexes[pHash->dwBlockIndex]; +                TMPQHash * pHashTableEnd = ha->pHashTable + ha->pHeader->dwHashTableSize; +                TMPQHash * pHash; -                    // Remap the old entry to the new entry -                    pNewTable[dwNewBlockIndex] = ha->pFileTable[pHash->dwBlockIndex]; -                    pHash->dwBlockIndex = dwNewBlockIndex; -                } -            } - -            // Replace the old file table with the new table -            STORM_FREE(ha->pFileTable); -            ha->pFileTable = pNewTable; -            ha->dwFileTableSize = dwNewTableSize; -        } - -        // Free the conversion table -        STORM_FREE(NewIndexes); -    } - -    return ERROR_SUCCESS; -} - -// Defragment the file table so it does not contain any gaps -// Note: As long as all values of all TMPQHash::dwBlockIndex -// are not HASH_ENTRY_FREE, the startup search index does not matter. -// Hash table is circular, so as long as there is no terminator, -// all entries will be found. -int DefragmentHashTable(TMPQArchive * ha) -{ -    TMPQHash * pNewTable; -    TMPQHash * pHashEnd = ha->pHashTable + ha->pHeader->dwHashTableSize; -    TMPQHash * pNewHash; -    TMPQHash * pHash; -    DWORD dwNewTableSize = 0; -    DWORD dwNewIndex = 0; - -    // Calculate how many entries in the hash table we really need -    for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) -        dwNewTableSize += IsValidHashEntry(ha, pHash) ? 1 : 0; -    dwNewTableSize = GetHashTableSizeForFileCount(dwNewTableSize); - -    // Could the hash table be shrinked? -    if(dwNewTableSize < ha->dwHashTableSize) -    { -        pNewTable = STORM_ALLOC(TMPQHash, dwNewTableSize); -        if(pNewTable != NULL) -        { -            // Initialize the new table -            memset(pNewTable, 0xFF, dwNewTableSize * sizeof(TMPQHash)); -            pNewHash = pNewTable; - -            // Copy the entries from the current hash table to new hash table -            for(pHash = ha->pHashTable; pHash < pHashEnd; pHash++) -            { -                if(IsValidHashEntry(ha, pHash)) +                for(pHash = ha->pHashTable; pHash < pHashTableEnd; pHash++)                  { -                    assert(dwNewIndex < dwNewTableSize); -                    pNewHash[dwNewIndex++] = pHash[0]; +                    if(pHash->dwBlockIndex < ha->dwFileTableSize) +                    { +                        assert(DefragmentTable[pHash->dwBlockIndex] != HASH_ENTRY_FREE); +                        pHash->dwBlockIndex = DefragmentTable[pHash->dwBlockIndex]; +                    }                  }              } - -            // Fill the remaining entries so they look like deleted -            while(dwNewIndex < dwNewTableSize) -                pNewHash[dwNewIndex++].dwBlockIndex = HASH_ENTRY_DELETED; - -            // Swap the hash tables -            STORM_FREE(ha->pHashTable); -            ha->pHashTable = pNewTable; -            ha->dwHashTableSize = dwNewTableSize;          } -    } -    return ERROR_SUCCESS; -} -// Performs final validation (and possible defragmentation) -// of malformed hash+block tables -int ShrinkMalformedMpqTables(TMPQArchive * ha) -{ -    // Sanity checks -    assert(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1); -    assert(ha->pHashTable != NULL); -    assert(ha->pFileTable != NULL); +        // Save the block table size +        ha->pHeader->dwBlockTableSize = ha->dwReservedFiles + dwBlockTableSize; -    // Now perform the defragmentation of the block table -    if(ha->dwFlags & MPQ_FLAG_BLOCK_TABLE_CUT) -    { -        assert(ha->dwFileTableSize == ha->pHeader->dwBlockTableSize); -        DefragmentFileTable(ha); -    } - -    // If the hash table has been cut, we need to defragment it -    // This will greatly improve performance on searching and loading listfile -    if(ha->dwFlags & MPQ_FLAG_HASH_TABLE_CUT) -    { -        assert(ha->dwHashTableSize == ha->pHeader->dwHashTableSize); -        DefragmentHashTable(ha); +        // Free the defragment table +        STORM_FREE(DefragmentTable);      } -/* -        TMPQHash * pSrcEntry = ha->pHashTable; -        TMPQHash * pTrgEntry = ha->pHashTable; -        DWORD dwNewEntryCount; -        DWORD dwNewTableSize; - -        assert(ha->dwHashTableSize == ha->pHeader->dwHashTableSize); - -        // Defragment the hash table -        for(i = 0; i < ha->dwHashTableSize; i++, pSrcEntry++) -        { -            // If the entry is valid, move it -            if(pSrcEntry->dwBlockIndex < ha->dwFileTableSize) -            { -                // Fix the hash table index in the file table and move the entry -                if(pTrgEntry != pSrcEntry) -                { -                    pFileTable[pSrcEntry->dwBlockIndex].dwHashIndex = (DWORD)(pTrgEntry - pHashTable); -                    *pTrgEntry = *pSrcEntry; -                } - -                // Increment the number of used entries -                pTrgEntry++; -            } -        } - -        // Calculate the new hash table size -        dwNewEntryCount = (DWORD)(pTrgEntry - pHashTable); -        dwNewTableSize = GetHashTableSizeForFileCount(dwNewEntryCount); - -        // Mark the extra entries in the hash table as deleted -        // This causes every hash search to go over the entire hash table, -        // but if the hash table is cut, it would do it anyway -        memset(pHashTable + dwNewEntryCount, 0xFF, (dwNewTableSize - dwNewEntryCount) * sizeof(TMPQHash)); -        for(i = dwNewEntryCount; i < dwNewTableSize; i++) -            pHashTable[i].dwBlockIndex = HASH_ENTRY_DELETED; -        // Change the hash table size in the header -        ha->dwHashTableSize = dwNewTableSize; -    } -*/      return ERROR_SUCCESS;  } @@ -2647,23 +2609,32 @@ int ShrinkMalformedMpqTables(TMPQArchive * ha)  int RebuildHetTable(TMPQArchive * ha)  {      TMPQHetTable * pOldHetTable = ha->pHetTable; -    ULONGLONG FileNameHash; -    DWORD i; +    TFileEntry * pFileTableEnd; +    TFileEntry * pFileEntry; +    DWORD dwBlockTableSize = ha->dwFileTableSize;      int nError = ERROR_SUCCESS; +    // If we are in the state of saving MPQ tables, the real size of block table +    // must already have been calculated. Use that value instead +    if(ha->dwFlags & MPQ_FLAG_SAVING_TABLES) +    { +        assert(ha->pHeader->dwBlockTableSize != 0); +        dwBlockTableSize = ha->pHeader->dwBlockTableSize; +    } +      // Create new HET table based on the total number of entries in the file table      // Note that if we fail to create it, we just stop using HET table -    ha->pHetTable = CreateHetTable(ha->dwFileTableSize, 0, 0x40, NULL); +    ha->pHetTable = CreateHetTable(dwBlockTableSize, 0, 0x40, NULL);      if(ha->pHetTable != NULL)      {          // Go through the file table again and insert all existing files -        for(i = 0; i < ha->dwFileTableSize; i++) +        pFileTableEnd = ha->pFileTable + dwBlockTableSize; +        for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++)          { -            if(ha->pFileTable[i].dwFlags & MPQ_FILE_EXISTS) +            if(pFileEntry->dwFlags & MPQ_FILE_EXISTS)              {                  // Get the high -                FileNameHash = ha->pFileTable[i].FileNameHash; -                nError = InsertHetEntry(ha->pHetTable, FileNameHash, i); +                nError = InsertHetEntry(ha->pHetTable, pFileEntry->FileNameHash, (DWORD)(pFileEntry - ha->pFileTable));                  if(nError != ERROR_SUCCESS)                      break;              } @@ -2677,31 +2648,33 @@ int RebuildHetTable(TMPQArchive * ha)  // Rebuilds the file table, removing all deleted file entries.  // Used when compacting the archive -int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize, DWORD dwNewMaxFileCount) +int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize)  { -    TFileEntry * pOldFileTableEnd = ha->pFileTable + ha->dwFileTableSize; -    TFileEntry * pOldFileTable = ha->pFileTable; -    TFileEntry * pFileTable;      TFileEntry * pFileEntry; -    TFileEntry * pOldEntry; +    TMPQHash * pHashTableEnd = ha->pHashTable + ha->pHeader->dwHashTableSize;      TMPQHash * pOldHashTable = ha->pHashTable;      TMPQHash * pHashTable = NULL;      TMPQHash * pHash;      int nError = ERROR_SUCCESS;      // The new hash table size must be greater or equal to the current hash table size -    assert(dwNewHashTableSize >= ha->dwHashTableSize); - -    // The new hash table size must be a power of two +    assert(dwNewHashTableSize >= ha->pHeader->dwHashTableSize); +    assert(dwNewHashTableSize >= ha->dwMaxFileCount);      assert((dwNewHashTableSize & (dwNewHashTableSize - 1)) == 0); +    assert(ha->pHashTable != NULL); -    // Allocate the new file table -    pFileTable = pFileEntry = STORM_ALLOC(TFileEntry, dwNewMaxFileCount); -    if(pFileTable == NULL) -        nError = ERROR_NOT_ENOUGH_MEMORY; +    // Reallocate the new file table, if needed +    if(dwNewHashTableSize > ha->dwFileTableSize) +    { +        ha->pFileTable = STORM_REALLOC(TFileEntry, ha->pFileTable, dwNewHashTableSize); +        if(ha->pFileTable == NULL) +            return ERROR_NOT_ENOUGH_MEMORY; + +        memset(ha->pFileTable + ha->dwFileTableSize, 0, (dwNewHashTableSize - ha->dwFileTableSize) * sizeof(TFileEntry)); +    }      // Allocate new hash table -    if(nError == ERROR_SUCCESS && ha->pHashTable != NULL) +    if(nError == ERROR_SUCCESS)      {          pHashTable = STORM_ALLOC(TMPQHash, dwNewHashTableSize);          if(pHashTable == NULL) @@ -2712,56 +2685,31 @@ int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize, DWORD dwNewMaxF      if(nError == ERROR_SUCCESS)      {          // Make sure that the hash table is properly filled -        memset(pFileTable, 0x00, sizeof(TFileEntry) * dwNewMaxFileCount);          memset(pHashTable, 0xFF, sizeof(TMPQHash) * dwNewHashTableSize); -         -        // Set the new tables to the MPQ archive -        ha->pFileTable = pFileTable;          ha->pHashTable = pHashTable;          // Set the new limits to the MPQ archive -        ha->dwHashTableSize = dwNewHashTableSize; -        ha->dwMaxFileCount = dwNewMaxFileCount; +        ha->pHeader->dwHashTableSize = dwNewHashTableSize; -        // Now copy all the file entries -        for(pOldEntry = pOldFileTable; pOldEntry < pOldFileTableEnd; pOldEntry++) +        // Parse the old hash table and copy all entries to the new table +        for(pHash = pOldHashTable; pHash < pHashTableEnd; pHash++)          { -            // If the file entry exists, we copy it to the new table -            // Otherwise, we skip it -            if(pOldEntry->dwFlags & MPQ_FILE_EXISTS) +            if(IsValidHashEntry2(ha, pHash))              { -                // Copy the file entry -                *pFileEntry = *pOldEntry; - -                // Create hash entry for it -                if(ha->pHashTable != NULL) -                { -                    pHash = AllocateHashEntry(ha, pFileEntry); -                    if(pHash == NULL) -                    { -                        nError = ERROR_DISK_FULL; -                        break; -                    } -                } - -                // Move the file entry by one -                pFileEntry++; +                pFileEntry = ha->pFileTable + pHash->dwBlockIndex; +                AllocateHashEntry(ha, pFileEntry, pHash->lcLocale);              }          } -        // Update the file table size -        ha->dwFileTableSize = (DWORD)(pFileEntry - pFileTable); +        // Increment the max file count for the file +        ha->dwFileTableSize = dwNewHashTableSize; +        ha->dwMaxFileCount = dwNewHashTableSize;          ha->dwFlags |= MPQ_FLAG_CHANGED; -        pFileTable = NULL;      }      // Now free the remaining entries -    if(pOldFileTable != NULL) -        STORM_FREE(pOldFileTable);      if(pOldHashTable != NULL)          STORM_FREE(pOldHashTable); -    if(pFileTable != NULL) -        STORM_FREE(pFileTable);      return nError;  } diff --git a/src/SFileAddFile.cpp b/src/SFileAddFile.cpp index aa47aa7..70a857b 100644 --- a/src/SFileAddFile.cpp +++ b/src/SFileAddFile.cpp @@ -354,7 +354,7 @@ static int RecryptFileData(  }  //----------------------------------------------------------------------------- -// Support functions for adding files to the MPQ +// Internal support for MPQ modifications  int SFileAddFile_Init(      TMPQArchive * ha, @@ -368,6 +368,7 @@ int SFileAddFile_Init(      TFileEntry * pFileEntry = NULL;      ULONGLONG TempPos;                  // For various file offset calculations      TMPQFile * hf = NULL;               // File structure for newly added file +    DWORD dwHashIndex = HASH_ENTRY_FREE;      int nError = ERROR_SUCCESS;      // @@ -410,7 +411,7 @@ int SFileAddFile_Init(          if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1)          {              TempPos  = hf->MpqFilePos + dwFileSize; -            TempPos += ha->dwHashTableSize * sizeof(TMPQHash); +            TempPos += ha->pHeader->dwHashTableSize * sizeof(TMPQHash);              TempPos += ha->dwFileTableSize * sizeof(TMPQBlock);              if((TempPos >> 32) != 0)                  nError = ERROR_DISK_FULL; @@ -421,25 +422,24 @@ int SFileAddFile_Init(      if(nError == ERROR_SUCCESS)      {          // Check if the file already exists in the archive -        pFileEntry = GetFileEntryExact(ha, szFileName, lcLocale); -        if(pFileEntry == NULL) -        { -            // First, free the internal files -            InvalidateInternalFiles(ha); - -            // Find a free entry in the file table -            pFileEntry = AllocateFileEntry(ha, szFileName, lcLocale); -            if(pFileEntry == NULL) -                nError = ERROR_DISK_FULL; -        } -        else +        pFileEntry = GetFileEntryExact(ha, szFileName, lcLocale, &dwHashIndex); +        if(pFileEntry != NULL)          { -            // If the caller didn't set MPQ_FILE_REPLACEEXISTING, fail it              if(dwFlags & MPQ_FILE_REPLACEEXISTING)                  InvalidateInternalFiles(ha);              else                  nError = ERROR_ALREADY_EXISTS;          } +        else +        { +            // Free all internal files - (listfile), (attributes), (signature) +            InvalidateInternalFiles(ha); +             +            // Attempt to allocate new file entry +            pFileEntry = AllocateFileEntry(ha, szFileName, lcLocale, &dwHashIndex); +            if(pFileEntry == NULL) +                nError = ERROR_DISK_FULL;    +        }      }      // Fill the file entry and TMPQFile structure @@ -452,7 +452,14 @@ int SFileAddFile_Init(          // Initialize the hash entry for the file          hf->pFileEntry = pFileEntry;          hf->dwDataSize = dwFileSize; -         + +        // Set the hash table entry +        if(ha->pHashTable != NULL && dwHashIndex < ha->pHeader->dwHashTableSize) +        { +            hf->pHashEntry = ha->pHashTable + dwHashIndex; +            hf->pHashEntry->lcLocale = (USHORT)lcLocale; +        } +          // Decrypt the file key          if(dwFlags & MPQ_FILE_ENCRYPTED)              hf->dwFileKey = DecryptFileKey(szFileName, hf->MpqFilePos, dwFileSize, dwFlags); @@ -462,7 +469,6 @@ int SFileAddFile_Init(          pFileEntry->dwFileSize = dwFileSize;          pFileEntry->dwCmpSize = 0;          pFileEntry->dwFlags  = dwFlags | MPQ_FILE_EXISTS; -        pFileEntry->lcLocale = (USHORT)lcLocale;          // Initialize the file time, CRC32 and MD5          assert(sizeof(hf->hctx) >= sizeof(hash_state)); @@ -668,7 +674,7 @@ int SFileAddFile_Finish(TMPQFile * hf)      {          // Free the file entry in MPQ tables          if(pFileEntry != NULL) -            DeleteFileEntry(ha, pFileEntry); +            DeleteFileEntry(ha, hf);      }      // Clear the add file callback @@ -1007,72 +1013,59 @@ bool WINAPI SFileAddWave(HANDLE hMpq, const TCHAR * szFileName, const char * szA  //-----------------------------------------------------------------------------  // bool SFileRemoveFile(HANDLE hMpq, char * szFileName)  // -// This function removes a file from the archive. The file content -// remains there, only the entries in the hash table and in the block -// table are updated.  +// This function removes a file from the archive. +//  bool WINAPI SFileRemoveFile(HANDLE hMpq, const char * szFileName, DWORD dwSearchScope)  { -    TMPQArchive * ha = (TMPQArchive *)hMpq; -    TFileEntry * pFileEntry = NULL; // File entry of the file to be deleted -    DWORD dwFileIndex = 0; +    TMPQArchive * ha = IsValidMpqHandle(hMpq); +    TMPQFile * hf = NULL;      int nError = ERROR_SUCCESS;      // Keep compiler happy      dwSearchScope = dwSearchScope;      // Check the parameters -    if(nError == ERROR_SUCCESS) -    { -        if(!IsValidMpqHandle(hMpq)) -            nError = ERROR_INVALID_HANDLE; -        if(szFileName == NULL || *szFileName == 0) -            nError = ERROR_INVALID_PARAMETER; -        if(IsInternalMpqFileName(szFileName)) -            nError = ERROR_INTERNAL_FILE; -    } +    if(ha == NULL) +        nError = ERROR_INVALID_HANDLE; +    if(szFileName == NULL || *szFileName == 0) +        nError = ERROR_INVALID_PARAMETER; +    if(IsInternalMpqFileName(szFileName)) +        nError = ERROR_INTERNAL_FILE; +    // Do not allow to remove files from read-only or patched MPQs      if(nError == ERROR_SUCCESS)      { -        // Do not allow to remove files from MPQ open for read only -        if(ha->dwFlags & MPQ_FLAG_READ_ONLY) +        if((ha->dwFlags & MPQ_FLAG_READ_ONLY) || (ha->haPatch != NULL))              nError = ERROR_ACCESS_DENIED;      } -    // Get hash entry belonging to this file +    // If all checks have passed, we can delete the file from the MPQ      if(nError == ERROR_SUCCESS)      { -        if(!IsPseudoFileName(szFileName, &dwFileIndex)) +        // Open the file from the MPQ +        if(SFileOpenFileEx(hMpq, szFileName, SFILE_OPEN_BASE_FILE, (HANDLE *)&hf))          { -            if((pFileEntry = GetFileEntryExact(ha, (char *)szFileName, lcFileLocale)) == NULL) -                nError = ERROR_FILE_NOT_FOUND; +            // Delete the file entry +            nError = DeleteFileEntry(ha, hf); +            FreeFileHandle(hf);          }          else -        { -            if((pFileEntry = GetFileEntryByIndex(ha, dwFileIndex)) == NULL) -                nError = ERROR_FILE_NOT_FOUND; -        } -    } - -    // Test if the file is not already deleted -    if(nError == ERROR_SUCCESS) -    { -        if(!(pFileEntry->dwFlags & MPQ_FILE_EXISTS)) -            nError = ERROR_FILE_NOT_FOUND; +            nError = GetLastError();      } +    // If the file has been deleted, we need to invalidate +    // the internal files and recreate HET table      if(nError == ERROR_SUCCESS)      { -        // Invalidate the entries for (listfile) and (attributes) +        // Invalidate the entries for internal files          // After we are done with MPQ changes, we need to re-create them anyway          InvalidateInternalFiles(ha); -        // Delete the file entry in the file table and defragment the file table -        DeleteFileEntry(ha, pFileEntry); - -        // We also need to rebuild the HET table, if present -        if(ha->pHetTable != NULL) -            nError = RebuildHetTable(ha); +        // +        // Don't rebuild HET table now; the file's flags indicate +        // that it's been deleted, which is enough +        //      }      // Resolve error and exit @@ -1084,81 +1077,56 @@ bool WINAPI SFileRemoveFile(HANDLE hMpq, const char * szFileName, DWORD dwSearch  // Renames the file within the archive.  bool WINAPI SFileRenameFile(HANDLE hMpq, const char * szFileName, const char * szNewFileName)  { -    TMPQArchive * ha = (TMPQArchive *)hMpq; -    TFileEntry * pFileEntry = NULL; -    ULONGLONG RawDataOffs; +    TMPQArchive * ha = IsValidMpqHandle(hMpq);      TMPQFile * hf;      int nError = ERROR_SUCCESS;      // Test the valid parameters -    if(nError == ERROR_SUCCESS) -    { -        if(!IsValidMpqHandle(hMpq)) -            nError = ERROR_INVALID_HANDLE; -        if(szFileName == NULL || *szFileName == 0 || szNewFileName == NULL || *szNewFileName == 0) -            nError = ERROR_INVALID_PARAMETER; -    } +    if(ha == NULL) +        nError = ERROR_INVALID_HANDLE; +    if(szFileName == NULL || *szFileName == 0 || szNewFileName == NULL || *szNewFileName == 0) +        nError = ERROR_INVALID_PARAMETER; +    if(IsInternalMpqFileName(szFileName) || IsInternalMpqFileName(szNewFileName)) +        nError = ERROR_INTERNAL_FILE; +    // Do not allow to rename files in MPQ open for read only      if(nError == ERROR_SUCCESS)      { -        // Do not allow to rename files in MPQ open for read only          if(ha->dwFlags & MPQ_FLAG_READ_ONLY)              nError = ERROR_ACCESS_DENIED; - -        // Do not allow renaming anything to a pseudo-file name -        if(IsPseudoFileName(szFileName, NULL) || IsPseudoFileName(szNewFileName, NULL)) -            nError = ERROR_INVALID_PARAMETER; - -        // Do not allow to rename any of the internal files -        // Also do not allow to rename any of files to an internal file -        if(IsInternalMpqFileName(szFileName) || IsInternalMpqFileName(szNewFileName)) -            nError = ERROR_INTERNAL_FILE;      } -    // Find the current file entry. +    // Open the new file. If exists, we don't allow rename operation      if(nError == ERROR_SUCCESS)      { -        // Get the file entry -        pFileEntry = GetFileEntryLocale(ha, szFileName, lcFileLocale); -        if(pFileEntry == NULL) -            nError = ERROR_FILE_NOT_FOUND; -    } -         -    // Also try to find file entry for the new file. -    // This verifies if we are not overwriting an existing file -    // (whose name we perhaps don't know) -    if(nError == ERROR_SUCCESS) -    { -        if(GetFileEntryLocale(ha, szNewFileName, pFileEntry->lcLocale) != NULL) +        if(GetFileEntryLocale(ha, szNewFileName, lcFileLocale) != NULL)              nError = ERROR_ALREADY_EXISTS;      } -    // Now we rename the existing file entry. +    // Open the file from the MPQ      if(nError == ERROR_SUCCESS)      { -        // Rename the file entry -        nError = RenameFileEntry(ha, pFileEntry, szNewFileName); -    } +        // Attempt to open the file +        if(SFileOpenFileEx(hMpq, szFileName, SFILE_OPEN_BASE_FILE, (HANDLE *)&hf)) +        { +            ULONGLONG RawDataOffs; +            TFileEntry * pFileEntry = hf->pFileEntry; -    // Now we need to recreate the HET table, if we have one -    if(nError == ERROR_SUCCESS && ha->pHetTable != NULL) -        nError = RebuildHetTable(ha); +            // Invalidate the entries for internal files +            InvalidateInternalFiles(ha); -    // Now we copy the existing file entry to the new one -    if(nError == ERROR_SUCCESS) -    { -        // If the file is encrypted, we have to re-crypt the file content -        // with the new decryption key -        if(pFileEntry->dwFlags & MPQ_FILE_ENCRYPTED) -        { -            hf = CreateFileHandle(ha, pFileEntry); -            if(hf != NULL) +            // Rename the file entry in the table +            nError = RenameFileEntry(ha, hf, szNewFileName); + +            // If the file is encrypted, we have to re-crypt the file content +            // with the new decryption key +            if((nError == ERROR_SUCCESS) && (pFileEntry->dwFlags & MPQ_FILE_ENCRYPTED))              {                  // Recrypt the file data in the MPQ                  nError = RecryptFileData(ha, hf, szFileName, szNewFileName); -                // Update the MD5 -                if(ha->pHeader->dwRawChunkSize != 0) +                // Update the MD5 of the raw block +                if(nError == ERROR_SUCCESS && ha->pHeader->dwRawChunkSize != 0)                  {                      RawDataOffs = ha->MpqPos + pFileEntry->ByteOffset;                      WriteMpqDataMD5(ha->pStream, @@ -1166,20 +1134,22 @@ bool WINAPI SFileRenameFile(HANDLE hMpq, const char * szFileName, const char * s                                      pFileEntry->dwCmpSize,                                      ha->pHeader->dwRawChunkSize);                  } -                 -                FreeFileHandle(hf); -            } -            else -            { -                nError = ERROR_NOT_ENOUGH_MEMORY;              } + +            // Free the file handle +            FreeFileHandle(hf); +        } +        else +        { +            nError = GetLastError();          }      } -    // Note: MPQ_FLAG_CHANGED is set by RenameFileEntry -    // assert((ha->dwFlags & MPQ_FLAG_CHANGED) != 0); +    // We also need to rebuild the HET table, if present +    if(nError == ERROR_SUCCESS && ha->pHetTable != NULL) +        nError = RebuildHetTable(ha); -    // Resolve error and return +    // Resolve error and exit      if(nError != ERROR_SUCCESS)          SetLastError(nError);      return (nError == ERROR_SUCCESS); @@ -1209,15 +1179,23 @@ bool WINAPI SFileSetFileLocale(HANDLE hFile, LCID lcNewLocale)  {      TMPQArchive * ha;      TFileEntry * pFileEntry; -    TMPQFile * hf = (TMPQFile *)hFile; +    TMPQFile * hf = IsValidFileHandle(hFile);      // Invalid handle => do nothing -    if(!IsValidFileHandle(hFile)) +    if(hf == NULL)      {          SetLastError(ERROR_INVALID_HANDLE);          return false;      } +    // Do not allow to rename files in MPQ open for read only +    ha = hf->ha; +    if(ha->dwFlags & MPQ_FLAG_READ_ONLY) +    { +        SetLastError(ERROR_ACCESS_DENIED); +        return false; +    } +      // Do not allow unnamed access      if(hf->pFileEntry->szFileName == NULL)      { @@ -1232,42 +1210,23 @@ bool WINAPI SFileSetFileLocale(HANDLE hFile, LCID lcNewLocale)          return false;      } -    // Do not allow changing file locales in MPQs version 3 or higher -    ha = hf->ha; -    if(ha->pHeader->wFormatVersion >= MPQ_FORMAT_VERSION_3) +    // Do not allow changing file locales if there is no hash table +    if(hf->pHashEntry == NULL)      {          SetLastError(ERROR_NOT_SUPPORTED);          return false;      } -    // Do not allow to rename files in MPQ open for read only -    if(ha->dwFlags & MPQ_FLAG_READ_ONLY) -    { -        SetLastError(ERROR_ACCESS_DENIED); -        return false; -    } - -    // If the file already has that locale, return OK -    if(hf->pFileEntry->lcLocale == lcNewLocale) -        return true; -      // We have to check if the file+locale is not already there -    pFileEntry = GetFileEntryExact(ha, hf->pFileEntry->szFileName, lcNewLocale); +    pFileEntry = GetFileEntryExact(ha, hf->pFileEntry->szFileName, lcNewLocale, NULL);      if(pFileEntry != NULL)      {          SetLastError(ERROR_ALREADY_EXISTS);          return false;      } -    // Set the locale and return success -    pFileEntry = hf->pFileEntry; -    pFileEntry->lcLocale = (USHORT)lcNewLocale; - -    // Save the new locale to the hash table, if any -    if(ha->pHashTable != NULL) -        ha->pHashTable[pFileEntry->dwHashIndex].lcLocale = (USHORT)lcNewLocale; - -    // Remember that the MPQ tables have been changed +    // Update the locale in the hash table entry +    hf->pHashEntry->lcLocale = (USHORT)lcNewLocale;      ha->dwFlags |= MPQ_FLAG_CHANGED;      return true;  } diff --git a/src/SFileAttributes.cpp b/src/SFileAttributes.cpp index 81bf1cf..dacfd3b 100644 --- a/src/SFileAttributes.cpp +++ b/src/SFileAttributes.cpp @@ -34,29 +34,29 @@ typedef struct _MPQ_ATTRIBUTES_HEADER  //-----------------------------------------------------------------------------  // Local functions -static DWORD GetSizeOfAttributesFile(DWORD dwAttrFlags, DWORD dwFileTableSize) +static DWORD GetSizeOfAttributesFile(DWORD dwAttrFlags, DWORD dwBlockTableSize)  {      DWORD cbAttrFile = sizeof(MPQ_ATTRIBUTES_HEADER);      // Calculate size of the (attributes) file      if(dwAttrFlags & MPQ_ATTRIBUTE_CRC32) -        cbAttrFile += dwFileTableSize * sizeof(DWORD); +        cbAttrFile += dwBlockTableSize * sizeof(DWORD);      if(dwAttrFlags & MPQ_ATTRIBUTE_FILETIME) -        cbAttrFile += dwFileTableSize * sizeof(ULONGLONG); +        cbAttrFile += dwBlockTableSize * sizeof(ULONGLONG);      if(dwAttrFlags & MPQ_ATTRIBUTE_MD5) -        cbAttrFile += dwFileTableSize * MD5_DIGEST_SIZE; +        cbAttrFile += dwBlockTableSize * MD5_DIGEST_SIZE;      // The bit array has been created without the last bit belonging to (attributes)      // When the number of files is a multiplier of 8 plus one, then the size of (attributes)      // if 1 byte less than expected.      // Example: wow-update-13164.MPQ: BlockTableSize = 0x62E1, but there's only 0xC5C bytes      if(dwAttrFlags & MPQ_ATTRIBUTE_PATCH_BIT) -        cbAttrFile += (dwFileTableSize + 6) / 8; +        cbAttrFile += (dwBlockTableSize + 6) / 8;      return cbAttrFile;  } -static DWORD CheckSizeOfAttributesFile(DWORD cbAttrFile, DWORD dwAttrFlags, DWORD dwFileTableSize) +static DWORD CheckSizeOfAttributesFile(DWORD cbAttrFile, DWORD dwAttrFlags, DWORD dwBlockTableSize)  {      DWORD cbHeaderSize = sizeof(MPQ_ATTRIBUTES_HEADER);      DWORD cbChecksumSize1 = 0; @@ -87,21 +87,21 @@ static DWORD CheckSizeOfAttributesFile(DWORD cbAttrFile, DWORD dwAttrFlags, DWOR      // Get the expected size of CRC32 array      if(dwAttrFlags & MPQ_ATTRIBUTE_CRC32)      { -        cbChecksumSize1 += dwFileTableSize * sizeof(DWORD); +        cbChecksumSize1 += dwBlockTableSize * sizeof(DWORD);          cbChecksumSize2 += cbChecksumSize1 - sizeof(DWORD);      }      // Get the expected size of FILETIME array      if(dwAttrFlags & MPQ_ATTRIBUTE_FILETIME)      { -        cbFileTimeSize1 += dwFileTableSize * sizeof(ULONGLONG); +        cbFileTimeSize1 += dwBlockTableSize * sizeof(ULONGLONG);          cbFileTimeSize2 += cbFileTimeSize1 - sizeof(ULONGLONG);      }      // Get the expected size of MD5 array      if(dwAttrFlags & MPQ_ATTRIBUTE_MD5)      { -        cbFileHashSize1 += dwFileTableSize * MD5_DIGEST_SIZE; +        cbFileHashSize1 += dwBlockTableSize * MD5_DIGEST_SIZE;          cbFileHashSize2 += cbFileHashSize1 - MD5_DIGEST_SIZE;      } @@ -109,26 +109,26 @@ static DWORD CheckSizeOfAttributesFile(DWORD cbAttrFile, DWORD dwAttrFlags, DWOR      if(dwAttrFlags & MPQ_ATTRIBUTE_PATCH_BIT)      {          cbPatchBitSize1 = -        cbPatchBitSize2 = ((dwFileTableSize + 6) / 8); -        cbPatchBitSize3 = dwFileTableSize * sizeof(DWORD); +        cbPatchBitSize2 = ((dwBlockTableSize + 6) / 8); +        cbPatchBitSize3 = dwBlockTableSize * sizeof(DWORD);      }      // Check if the (attributes) file entry count is equal to our file table size      if(cbAttrFile == (cbHeaderSize + cbChecksumSize1 + cbFileTimeSize1 + cbFileHashSize1 + cbPatchBitSize1)) -        return dwFileTableSize; +        return dwBlockTableSize;      // Check if the (attributes) file entry count is equal to our file table size minus one      if(cbAttrFile == (cbHeaderSize + cbChecksumSize2 + cbFileTimeSize2 + cbFileHashSize2 + cbPatchBitSize2)) -        return dwFileTableSize - 1; +        return dwBlockTableSize - 1;      // Zenith.SC2MAP has the MPQ_ATTRIBUTE_PATCH_BIT set, but the bit array is missing      if(cbAttrFile == (cbHeaderSize + cbChecksumSize1 + cbFileTimeSize1 + cbFileHashSize1)) -        return dwFileTableSize; +        return dwBlockTableSize;      // interface.MPQ.part (WoW build 10958) has the MPQ_ATTRIBUTE_PATCH_BIT set      // but there's an array of DWORDs (filled with zeros) instead of array of bits      if(cbAttrFile == (cbHeaderSize + cbChecksumSize1 + cbFileTimeSize1 + cbFileHashSize1 + cbPatchBitSize3)) -        return dwFileTableSize; +        return dwBlockTableSize;  #ifdef __STORMLIB_TEST__      // Invalid size of the (attributes) file @@ -246,12 +246,11 @@ static int LoadAttributesFile(TMPQArchive * ha, LPBYTE pbAttrFile, DWORD cbAttrF  static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile)  {      PMPQ_ATTRIBUTES_HEADER pAttrHeader; -    TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; +    TFileEntry * pFileTableEnd = ha->pFileTable + ha->pHeader->dwBlockTableSize;      TFileEntry * pFileEntry;      LPBYTE pbAttrFile;      LPBYTE pbAttrPtr;      size_t cbAttrFile; -    DWORD dwFinalEntries = ha->dwFileTableSize + ha->dwReservedFiles;      // Check if we need patch bits in the (attributes) file      for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++) @@ -265,7 +264,7 @@ static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile)      // Allocate the buffer for holding the entire (attributes)      // Allocate 1 byte more (See GetSizeOfAttributesFile for more info) -    cbAttrFile = GetSizeOfAttributesFile(ha->dwAttrFlags, dwFinalEntries); +    cbAttrFile = GetSizeOfAttributesFile(ha->dwAttrFlags, ha->pHeader->dwBlockTableSize);      pbAttrFile = pbAttrPtr = STORM_ALLOC(BYTE, cbAttrFile + 1);      if(pbAttrFile != NULL)      { @@ -287,8 +286,8 @@ static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile)              for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++)                  *pArrayCRC32++ = BSWAP_INT32_UNSIGNED(pFileEntry->dwCrc32); -            // Skip the reserved entries -            pbAttrPtr = (LPBYTE)(pArrayCRC32 + ha->dwReservedFiles); +            // Update pointer +            pbAttrPtr = (LPBYTE)pArrayCRC32;          }          // Write the array of file time @@ -300,8 +299,8 @@ static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile)              for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++)                  *pArrayFileTime++ = BSWAP_INT64_UNSIGNED(pFileEntry->FileTime); -            // Skip the reserved entries -            pbAttrPtr = (LPBYTE)(pArrayFileTime + ha->dwReservedFiles); +            // Update pointer +            pbAttrPtr = (LPBYTE)pArrayFileTime;          }          // Write the array of MD5s @@ -316,8 +315,8 @@ static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile)                  pbArrayMD5 += MD5_DIGEST_SIZE;              } -            // Skip the reserved items -            pbAttrPtr = pbArrayMD5 + (ha->dwReservedFiles * MD5_DIGEST_SIZE); +            // Update pointer +            pbAttrPtr = pbArrayMD5;          }          // Write the array of patch bits @@ -339,12 +338,8 @@ static LPBYTE CreateAttributesFile(TMPQArchive * ha, DWORD * pcbAttrFile)                  dwBitMask = (dwBitMask << 0x07) | (dwBitMask >> 0x01);              } -            // Having incremented the bit array just by the number of items in the file table -            // will create the bit array one byte less of the number of files is a multiplier of 8). -            // Blizzard MPQs have the same feature. -              // Move past the bit array -            pbAttrPtr = (pbBitArray + dwByteIndex) + ((dwBitMask & 0x7F) ? 1 : 0); +            pbAttrPtr += (ha->pHeader->dwBlockTableSize + 6) / 8;          }          // Now we expect that current position matches the estimated size @@ -419,7 +414,7 @@ int SAttrFileSaveToMpq(TMPQArchive * ha)      if(ha->dwFileFlags2 != 0)      {          // At this point, we expect to have at least one reserved entry in the file table -        assert(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID); +        assert(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_NEW);          assert(ha->dwReservedFiles > 0);          // Create the raw data that is to be written to (attributes) @@ -451,6 +446,10 @@ int SAttrFileSaveToMpq(TMPQArchive * ha)                  SFileAddFile_Finish(hf);              } +            // Clear the number of reserved files +            ha->dwFlags &= ~(MPQ_FLAG_ATTRIBUTES_NEW | MPQ_FLAG_ATTRIBUTES_NONE); +            ha->dwReservedFiles--; +              // Free the attributes buffer              STORM_FREE(pbAttrFile);          } @@ -459,10 +458,6 @@ int SAttrFileSaveToMpq(TMPQArchive * ha)              // If the (attributes) file would be empty, its OK              nError = (cbAttrFile == 0) ? ERROR_SUCCESS : ERROR_NOT_ENOUGH_MEMORY;          } - -        // Clear the number of reserved files -        ha->dwFlags &= ~MPQ_FLAG_ATTRIBUTES_INVALID; -        ha->dwReservedFiles--;      }      return nError; diff --git a/src/SFileCompactArchive.cpp b/src/SFileCompactArchive.cpp index a70fa0a..2c0ef5a 100644 --- a/src/SFileCompactArchive.cpp +++ b/src/SFileCompactArchive.cpp @@ -598,18 +598,10 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR      if(nError == ERROR_SUCCESS)          nError = CopyMpqFiles(ha, pFileKeys, pTempStream); -    // We also need to rebuild the HET table, if any -    if(nError == ERROR_SUCCESS) -    { -        // Rebuild the HET table, if we have any -        if(ha->pHetTable != NULL) -            nError = RebuildHetTable(ha); -        ha->dwFlags |= MPQ_FLAG_CHANGED; -    } -      // If succeeded, switch the streams      if(nError == ERROR_SUCCESS)      { +        ha->dwFlags |= MPQ_FLAG_CHANGED;          if(FileStream_Replace(ha->pStream, pTempStream))              pTempStream = NULL;          else @@ -619,7 +611,7 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR      // Final user notification      if(nError == ERROR_SUCCESS && ha->pfnCompactCB != NULL)      { -        ha->CompactBytesProcessed += (ha->dwHashTableSize * sizeof(TMPQHash)); +        ha->CompactBytesProcessed += (ha->pHeader->dwHashTableSize * sizeof(TMPQHash));          ha->CompactBytesProcessed += (ha->dwFileTableSize * sizeof(TMPQBlock));          ha->pfnCompactCB(ha->pvCompactUserData, CCB_CLOSING_ARCHIVE, ha->CompactBytesProcessed, ha->CompactTotalBytes);      } @@ -658,21 +650,18 @@ bool WINAPI SFileSetMaxFileCount(HANDLE hMpq, DWORD dwMaxFileCount)      if(dwMaxFileCount < ha->dwFileTableSize)          nError = ERROR_DISK_FULL; -    // ALL file names must be known in order to be able -    // to rebuild hash table -    if(nError == ERROR_SUCCESS) +    // ALL file names must be known in order to be able to rebuild hash table +    if(nError == ERROR_SUCCESS && ha->pHashTable != NULL)      {          nError = CheckIfAllFilesKnown(ha); -    } - -    // If the MPQ has a hash table, then we relocate the hash table -    if(nError == ERROR_SUCCESS) -    { -        // Calculate the hash table size for the new file limit -        dwNewHashTableSize = GetHashTableSizeForFileCount(dwMaxFileCount); +        if(nError == ERROR_SUCCESS) +        { +            // Calculate the hash table size for the new file limit +            dwNewHashTableSize = GetHashTableSizeForFileCount(dwMaxFileCount); -        // Rebuild both file tables -        nError = RebuildFileTable(ha, dwNewHashTableSize, dwMaxFileCount); +            // Rebuild both file tables +            nError = RebuildFileTable(ha, dwNewHashTableSize); +        }      }      // We always have to rebuild the (attributes) file due to file table change diff --git a/src/SFileCreateArchive.cpp b/src/SFileCreateArchive.cpp index 206baff..fb9ed60 100644 --- a/src/SFileCreateArchive.cpp +++ b/src/SFileCreateArchive.cpp @@ -159,21 +159,21 @@ bool WINAPI SFileCreateArchive2(const TCHAR * szMpqName, PSFILE_CREATE_MPQ pCrea      // Increment the maximum amount of files to have space for (listfile)      if(pCreateInfo->dwMaxFileCount && pCreateInfo->dwFileFlags1)      { -        dwMpqFlags |= MPQ_FLAG_LISTFILE_INVALID; +        dwMpqFlags |= MPQ_FLAG_LISTFILE_NEW;          dwReservedFiles++;      }      // Increment the maximum amount of files to have space for (attributes)      if(pCreateInfo->dwMaxFileCount && pCreateInfo->dwFileFlags2 && pCreateInfo->dwAttrFlags)      { -        dwMpqFlags |= MPQ_FLAG_ATTRIBUTES_INVALID; +        dwMpqFlags |= MPQ_FLAG_ATTRIBUTES_NEW;          dwReservedFiles++;      }      // Increment the maximum amount of files to have space for (signature)      if(pCreateInfo->dwMaxFileCount && pCreateInfo->dwFileFlags3)      { -        dwMpqFlags |= MPQ_FLAG_SIGNATURE_INVALID; +        dwMpqFlags |= MPQ_FLAG_SIGNATURE_NEW;          dwReservedFiles++;      } @@ -256,11 +256,7 @@ bool WINAPI SFileCreateArchive2(const TCHAR * szMpqName, PSFILE_CREATE_MPQ pCrea      // Create initial file table      if(nError == ERROR_SUCCESS && ha->dwMaxFileCount != 0)      { -        ha->pFileTable = STORM_ALLOC(TFileEntry, ha->dwMaxFileCount); -        if(ha->pFileTable != NULL) -            memset(ha->pFileTable, 0x00, sizeof(TFileEntry) * ha->dwMaxFileCount); -        else -            nError = ERROR_NOT_ENOUGH_MEMORY; +        nError = CreateFileTable(ha, ha->dwMaxFileCount);      }      // Cleanup : If an error, delete all buffers and return diff --git a/src/SFileFindFile.cpp b/src/SFileFindFile.cpp index fb71871..198f960 100644 --- a/src/SFileFindFile.cpp +++ b/src/SFileFindFile.cpp @@ -185,7 +185,6 @@ static TFileEntry * FindPatchEntry(TMPQArchive * ha, TFileEntry * pFileEntry)      TFileEntry * pPatchEntry = NULL;      TFileEntry * pTempEntry;      char szFileName[MAX_PATH]; -    LCID lcLocale = pFileEntry->lcLocale;      // Go while there are patches      while(ha->haPatch != NULL) @@ -200,7 +199,7 @@ static TFileEntry * FindPatchEntry(TMPQArchive * ha, TFileEntry * pFileEntry)          strcat(szFileName, pFileEntry->szFileName);          // Try to find the file there -        pTempEntry = GetFileEntryExact(ha, szFileName, lcLocale); +        pTempEntry = GetFileEntryExact(ha, szFileName, 0, NULL);          if(pTempEntry != NULL)              pPatchEntry = pTempEntry;      } @@ -218,7 +217,7 @@ static int DoMPQSearch(TMPQSearch * hs, SFILE_FIND_DATA * lpFindFileData)      TFileEntry * pFileEntry;      const char * szFileName;      HANDLE hFile; -    char szPseudoName[20]; +    char szNameBuff[MAX_PATH];      DWORD dwBlockIndex;      size_t nPrefixLength; @@ -260,11 +259,11 @@ static int DoMPQSearch(TMPQSearch * hs, SFILE_FIND_DATA * lpFindFileData)                      if(szFileName == NULL)                      {                          // Open the file by its pseudo-name. -                        // This also generates the file name with a proper extension -                        sprintf(szPseudoName, "File%08u.xxx", (unsigned int)dwBlockIndex); -                        if(SFileOpenFileEx((HANDLE)hs->ha, szPseudoName, SFILE_OPEN_BASE_FILE, &hFile)) +                        sprintf(szNameBuff, "File%08u.xxx", (unsigned int)dwBlockIndex); +                        if(SFileOpenFileEx((HANDLE)hs->ha, szNameBuff, SFILE_OPEN_BASE_FILE, &hFile))                          { -                            szFileName = (pFileEntry->szFileName != NULL) ? pFileEntry->szFileName : szPseudoName; +                            SFileGetFileName(hFile, szNameBuff); +                            szFileName = szNameBuff;                              SFileCloseFile(hFile);                          }                      } @@ -276,12 +275,11 @@ static int DoMPQSearch(TMPQSearch * hs, SFILE_FIND_DATA * lpFindFileData)                          if(CheckWildCard(szFileName + nPrefixLength, hs->szSearchMask))                          {                              // Fill the found entry. hash entry and block index are taken from the base MPQ -                            lpFindFileData->dwHashIndex  = pFileEntry->dwHashIndex;                              lpFindFileData->dwBlockIndex = dwBlockIndex;                              lpFindFileData->dwFileSize   = pPatchEntry->dwFileSize;                              lpFindFileData->dwFileFlags  = pPatchEntry->dwFlags;                              lpFindFileData->dwCompSize   = pPatchEntry->dwCmpSize; -                            lpFindFileData->lcLocale     = pPatchEntry->lcLocale; +                            lpFindFileData->lcLocale     = 0;   // pPatchEntry->lcLocale;                              // Fill the filetime                              lpFindFileData->dwFileTimeHi = (DWORD)(pPatchEntry->FileTime >> 32); diff --git a/src/SFileGetFileInfo.cpp b/src/SFileGetFileInfo.cpp index f7faea0..40b2720 100644 --- a/src/SFileGetFileInfo.cpp +++ b/src/SFileGetFileInfo.cpp @@ -143,7 +143,6 @@ bool WINAPI SFileGetFileInfo(      TFileEntry * pFileEntry = NULL;      ULONGLONG Int64Value = 0;      ULONGLONG ByteOffset = 0; -    TMPQHash * pHash;      TMPQFile * hf = NULL;      void * pvSrcFileInfo = NULL;      DWORD cbSrcFileInfo = 0; @@ -376,7 +375,7 @@ bool WINAPI SFileGetFileInfo(              if(ha != NULL && ha->pHashTable != NULL)              {                  pvSrcFileInfo = ha->pHashTable; -                cbSrcFileInfo = ha->dwHashTableSize * sizeof(TMPQHash); +                cbSrcFileInfo = ha->pHeader->dwHashTableSize * sizeof(TMPQHash);                  nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER;              }              break; @@ -629,9 +628,9 @@ bool WINAPI SFileGetFileInfo(          case SFileInfoHashEntry:              hf = IsValidFileHandle(hMpqOrFile); -            if(hf != NULL && hf->ha != NULL && hf->ha->pHashTable != NULL) +            if(hf != NULL && hf->pHashEntry != NULL)              { -                pvSrcFileInfo = hf->ha->pHashTable + hf->pFileEntry->dwHashIndex; +                pvSrcFileInfo = hf->pHashEntry;                  cbSrcFileInfo = sizeof(TMPQHash);                  nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER;              } @@ -639,9 +638,9 @@ bool WINAPI SFileGetFileInfo(          case SFileInfoHashIndex:              hf = IsValidFileHandle(hMpqOrFile); -            if(hf != NULL && hf->pFileEntry != NULL) +            if(hf != NULL && hf->pHashEntry != NULL)              { -                pvSrcFileInfo = &hf->pFileEntry->dwHashIndex; +                pvSrcFileInfo = &hf->dwHashIndex;                  cbSrcFileInfo = sizeof(DWORD);                  nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER;              } @@ -649,10 +648,10 @@ bool WINAPI SFileGetFileInfo(          case SFileInfoNameHash1:              hf = IsValidFileHandle(hMpqOrFile); -            if(hf != NULL && hf->ha != NULL && hf->ha->pHashTable != NULL) +            if(hf != NULL && hf->pHashEntry != NULL)              { -                pHash = hf->ha->pHashTable + hf->pFileEntry->dwHashIndex; -                pvSrcFileInfo = &pHash->dwName1; +                dwInt32Value = hf->pHashEntry->dwName1; +                pvSrcFileInfo = &dwInt32Value;                  cbSrcFileInfo = sizeof(DWORD);                  nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER;              } @@ -660,10 +659,10 @@ bool WINAPI SFileGetFileInfo(          case SFileInfoNameHash2:              hf = IsValidFileHandle(hMpqOrFile); -            if(hf != NULL && hf->ha != NULL && hf->ha->pHashTable != NULL) +            if(hf != NULL && hf->pHashEntry != NULL)              { -                pHash = hf->ha->pHashTable + hf->pFileEntry->dwHashIndex; -                pvSrcFileInfo = &pHash->dwName2; +                dwInt32Value = hf->pHashEntry->dwName2; +                pvSrcFileInfo = &dwInt32Value;                  cbSrcFileInfo = sizeof(DWORD);                  nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER;              } @@ -681,9 +680,9 @@ bool WINAPI SFileGetFileInfo(          case SFileInfoLocale:              hf = IsValidFileHandle(hMpqOrFile); -            if(hf != NULL && hf->pFileEntry != NULL) +            if(hf != NULL && hf->pHashEntry != NULL)              { -                dwInt32Value = hf->pFileEntry->lcLocale; +                dwInt32Value = hf->pHashEntry->lcLocale;                  pvSrcFileInfo = &dwInt32Value;                  cbSrcFileInfo = sizeof(DWORD);                  nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER; @@ -950,10 +949,6 @@ bool WINAPI SFileGetFileName(HANDLE hFile, char * szFileName)      TMPQFile * hf = (TMPQFile *)hFile;  // MPQ File handle      int nError = ERROR_INVALID_HANDLE; -    // Pre-zero the output buffer -    if(szFileName != NULL) -        *szFileName = 0; -      // Check valid parameters      if(IsValidFileHandle(hFile))      { @@ -966,15 +961,11 @@ bool WINAPI SFileGetFileName(HANDLE hFile, char * szFileName)              {                  // If the file name is not there yet, create a pseudo name                  if(pFileEntry->szFileName == NULL) -                {                      nError = CreatePseudoFileName(hFile, pFileEntry, szFileName); -                } -                else -                { -                    if(szFileName != NULL) -                        strcpy(szFileName, pFileEntry->szFileName); -                    nError = ERROR_SUCCESS; -                } + +                // Copy the file name to the output buffer, if any +                if(pFileEntry->szFileName && szFileName) +                    strcpy(szFileName, pFileEntry->szFileName);              }          } diff --git a/src/SFileListFile.cpp b/src/SFileListFile.cpp index 5f8a9df..385098e 100644 --- a/src/SFileListFile.cpp +++ b/src/SFileListFile.cpp @@ -389,7 +389,7 @@ static int SListFileCreateNodeForAllLocales(TMPQArchive * ha, const char * szFil      // If we have HET table, use that one      if(ha->pHetTable != NULL)      { -        pFileEntry = GetFileEntryAny(ha, szFileName); +        pFileEntry = GetFileEntryLocale(ha, szFileName, 0);          if(pFileEntry != NULL)          {              // Allocate file name for the file entry @@ -402,18 +402,12 @@ static int SListFileCreateNodeForAllLocales(TMPQArchive * ha, const char * szFil      // If we have hash table, we use it      if(ha->pHashTable != NULL)      { -        // Look for the first hash table entry for the file -        pFirstHash = pHash = GetFirstHashEntry(ha, szFileName); -          // Go while we found something +        pFirstHash = pHash = GetFirstHashEntry(ha, szFileName);          while(pHash != NULL)          { -            // Is it a valid file table index ? -            if(pHash->dwBlockIndex < ha->dwFileTableSize) -            { -                // Allocate file name for the file entry -                AllocateFileName(ha, ha->pFileTable + pHash->dwBlockIndex, szFileName); -            } +            // Allocate file name for the file entry +            AllocateFileName(ha, ha->pFileTable + pHash->dwBlockIndex, szFileName);              // Now find the next language version of the file              pHash = GetNextHashEntry(ha, pFirstHash, pHash); @@ -437,7 +431,7 @@ int SListFileSaveToMpq(TMPQArchive * ha)      if(ha->dwFileFlags1 != 0)      {          // At this point, we expect to have at least one reserved entry in the file table -        assert(ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID); +        assert(ha->dwFlags & MPQ_FLAG_LISTFILE_NEW);          assert(ha->dwReservedFiles > 0);          // Create the raw data that is to be written to (listfile) @@ -470,6 +464,10 @@ int SListFileSaveToMpq(TMPQArchive * ha)                  SFileAddFile_Finish(hf);              } +            // Clear the listfile flags +            ha->dwFlags &= ~(MPQ_FLAG_LISTFILE_NEW | MPQ_FLAG_LISTFILE_NONE); +            ha->dwReservedFiles--; +              // Free the listfile buffer              STORM_FREE(pbListFile);          } @@ -478,10 +476,6 @@ int SListFileSaveToMpq(TMPQArchive * ha)              // If the (listfile) file would be empty, its OK              nError = (cbListFile == 0) ? ERROR_SUCCESS : ERROR_NOT_ENOUGH_MEMORY;          } - -        // Clear the listfile flags -        ha->dwFlags &= ~MPQ_FLAG_LISTFILE_INVALID; -        ha->dwReservedFiles--;      }      return nError; diff --git a/src/SFileOpenArchive.cpp b/src/SFileOpenArchive.cpp index ffd5bfe..5ce4290 100644 --- a/src/SFileOpenArchive.cpp +++ b/src/SFileOpenArchive.cpp @@ -339,8 +339,9 @@ bool WINAPI SFileOpenArchive(                  ha->UserDataPos = SearchOffset;              // Set the position of the MPQ header -            ha->pHeader = (TMPQHeader *)ha->HeaderData; -            ha->MpqPos = SearchOffset; +            ha->pHeader  = (TMPQHeader *)ha->HeaderData; +            ha->MpqPos   = SearchOffset; +            ha->FileSize = FileSize;              // Sector size must be nonzero.              if(SearchOffset >= FileSize || ha->pHeader->wSectorSize == 0) @@ -395,14 +396,6 @@ bool WINAPI SFileOpenArchive(          nError = BuildFileTable(ha);      } -    // We now have both hash table and block table loaded. -    // If any malformation was detected in the MPQ, -    // we perform hash table and block table validation (and eventual defragmentation) -    if(nError == ERROR_SUCCESS && ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1 && (ha->dwFlags & MPQ_FLAG_MALFORMED)) -    { -        nError = ShrinkMalformedMpqTables(ha); -    } -      // Load the internal listfile and include it to the file table      if(nError == ERROR_SUCCESS && (dwFlags & MPQ_OPEN_NO_LISTFILE) == 0)      { @@ -492,7 +485,6 @@ bool WINAPI SFileSetDownloadCallback(HANDLE hMpq, SFILE_DOWNLOAD_CALLBACK Downlo  bool WINAPI SFileFlushArchive(HANDLE hMpq)  { -    TFileEntry * pFileEntry;      TMPQArchive * ha;      int nResultError = ERROR_SUCCESS;      int nError; @@ -510,44 +502,7 @@ bool WINAPI SFileFlushArchive(HANDLE hMpq)          // Indicate that we are saving MPQ internal structures          ha->dwFlags |= MPQ_FLAG_SAVING_TABLES; -        // -        // Invalidate entries for each internal file -        // - -        if(ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) -        { -            pFileEntry = GetFileEntryExact(ha, LISTFILE_NAME, LANG_NEUTRAL); -            if(pFileEntry != NULL) -            { -                DeleteFileEntry(ha, pFileEntry); -                ha->dwReservedFiles++; -            } -        } - -        if(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) -        { -            pFileEntry = GetFileEntryExact(ha, ATTRIBUTES_NAME, LANG_NEUTRAL); -            if(pFileEntry != NULL) -            { -                DeleteFileEntry(ha, pFileEntry); -                ha->dwReservedFiles++; -            } -        } - -        if(ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) -        { -            pFileEntry = GetFileEntryExact(ha, SIGNATURE_NAME, LANG_NEUTRAL); -            if(pFileEntry != NULL) -            { -                DeleteFileEntry(ha, pFileEntry); -                ha->dwReservedFiles++; -            } -        } - -        //          // Defragment the file table. This will allow us to put the internal files to the end -        // -          DefragmentFileTable(ha);          // @@ -555,21 +510,21 @@ bool WINAPI SFileFlushArchive(HANDLE hMpq)          // Note that the (signature) file is usually before (listfile) in the file table          // -        if(ha->dwFlags & MPQ_FLAG_SIGNATURE_INVALID) +        if(ha->dwFlags & MPQ_FLAG_SIGNATURE_NEW)          {              nError = SSignFileCreate(ha);              if(nError != ERROR_SUCCESS)                  nResultError = nError;          } -        if(ha->dwFlags & MPQ_FLAG_LISTFILE_INVALID) +        if(ha->dwFlags & MPQ_FLAG_LISTFILE_NEW)          {              nError = SListFileSaveToMpq(ha);              if(nError != ERROR_SUCCESS)                  nResultError = nError;          } -        if(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_INVALID) +        if(ha->dwFlags & MPQ_FLAG_ATTRIBUTES_NEW)          {              nError = SAttrFileSaveToMpq(ha);              if(nError != ERROR_SUCCESS) @@ -579,6 +534,10 @@ bool WINAPI SFileFlushArchive(HANDLE hMpq)          // Save HET table, BET table, hash table, block table, hi-block table          if(ha->dwFlags & MPQ_FLAG_CHANGED)          { +            // Rebuild the HET table +            if(ha->pHetTable != NULL) +                RebuildHetTable(ha); +              // Save all MPQ tables first              nError = SaveMPQTables(ha);              if(nError != ERROR_SUCCESS) diff --git a/src/SFileOpenFileEx.cpp b/src/SFileOpenFileEx.cpp index 3a9976e..3794680 100644 --- a/src/SFileOpenFileEx.cpp +++ b/src/SFileOpenFileEx.cpp @@ -16,6 +16,33 @@  /* Local functions                                                           */  /*****************************************************************************/ +static DWORD FindHashIndex(TMPQArchive * ha, DWORD dwFileIndex) +{ +    TMPQHash * pHashTableEnd; +    TMPQHash * pHash; +    DWORD dwFirstIndex = HASH_ENTRY_FREE; + +    // Should only be called if the archive has hash table +    assert(ha->pHashTable != NULL); + +    // Multiple hash table entries can point to the file table entry. +    // We need to search all of them +    pHashTableEnd = ha->pHashTable + ha->pHeader->dwHashTableSize; +    for(pHash = ha->pHashTable; pHash < pHashTableEnd; pHash++) +    { +        if(pHash->dwBlockIndex == dwFileIndex) +        { +            // Duplicate hash entry found +            if(dwFirstIndex != HASH_ENTRY_FREE) +                return HASH_ENTRY_FREE; +            dwFirstIndex = (DWORD)(pHash - ha->pHashTable); +        } +    } + +    // Return the hash table entry index +    return dwFirstIndex; +} +  static const char * GetPatchFileName(TMPQArchive * ha, const char * szFileName, char * szBuffer)  {      TMPQNamePrefix * pPrefix; @@ -41,7 +68,7 @@ static const char * GetPatchFileName(TMPQArchive * ha, const char * szFileName,      return szFileName;  } -static bool OpenLocalFile(const char * szFileName, HANDLE * phFile) +static bool OpenLocalFile(const char * szFileName, HANDLE * PtrFile)  {      TFileStream * pStream;      TMPQFile * hf = NULL; @@ -59,7 +86,7 @@ static bool OpenLocalFile(const char * szFileName, HANDLE * phFile)          if(hf != NULL)          {              hf->pStream = pStream; -            *phFile = hf; +            *PtrFile = hf;              return true;          }          else @@ -68,11 +95,11 @@ static bool OpenLocalFile(const char * szFileName, HANDLE * phFile)              SetLastError(ERROR_NOT_ENOUGH_MEMORY);          }      } -    *phFile = NULL; +    *PtrFile = NULL;      return false;  } -bool OpenPatchedFile(HANDLE hMpq, const char * szFileName, HANDLE * phFile) +bool OpenPatchedFile(HANDLE hMpq, const char * szFileName, HANDLE * PtrFile)  {      TMPQArchive * haBase = NULL;      TMPQArchive * ha = (TMPQArchive *)hMpq; @@ -88,7 +115,7 @@ bool OpenPatchedFile(HANDLE hMpq, const char * szFileName, HANDLE * phFile)      while(ha != NULL)      {          // If the file is there, then we remember the archive -        pFileEntry = GetFileEntryExact(ha, GetPatchFileName(ha, szFileName, szNameBuffer), 0); +        pFileEntry = GetFileEntryExact(ha, GetPatchFileName(ha, szFileName, szNameBuffer), 0, NULL);          if(pFileEntry != NULL && (pFileEntry->dwFlags & MPQ_FILE_PATCH_FILE) == 0)              haBase = ha; @@ -130,8 +157,8 @@ bool OpenPatchedFile(HANDLE hMpq, const char * szFileName, HANDLE * phFile)      }      // Give the updated base MPQ -    if(phFile != NULL) -        *phFile = (HANDLE)hfBase; +    if(PtrFile != NULL) +        *PtrFile = (HANDLE)hfBase;      return (hfBase != NULL);  } @@ -148,15 +175,15 @@ bool OpenPatchedFile(HANDLE hMpq, const char * szFileName, HANDLE * phFile)  int WINAPI SFileEnumLocales(      HANDLE hMpq,      const char * szFileName, -    LCID * plcLocales, -    LPDWORD pdwMaxLocales, +    LCID * PtrLocales, +    LPDWORD PtrMaxLocales,      DWORD dwSearchScope)  {      TMPQArchive * ha = (TMPQArchive *)hMpq; -    TFileEntry * pFileEntry;      TMPQHash * pFirstHash;      TMPQHash * pHash;      DWORD dwFileIndex = 0; +    DWORD dwMaxLocales;      DWORD dwLocales = 0;      // Test the parameters @@ -164,130 +191,63 @@ int WINAPI SFileEnumLocales(          return ERROR_INVALID_HANDLE;      if(szFileName == NULL || *szFileName == 0)          return ERROR_INVALID_PARAMETER; -    if(pdwMaxLocales == NULL) +    if(ha->pHashTable == NULL) +        return ERROR_NOT_SUPPORTED; +    if(PtrMaxLocales == NULL) +        return ERROR_INVALID_PARAMETER; +    if(IsPseudoFileName(szFileName, &dwFileIndex))          return ERROR_INVALID_PARAMETER;      // Keep compiler happy +    dwMaxLocales = PtrMaxLocales[0];      dwSearchScope = dwSearchScope; -    // Parse hash table entries for all locales -    if(!IsPseudoFileName(szFileName, &dwFileIndex)) -    { -        // Calculate the number of locales -        pFirstHash = pHash = GetFirstHashEntry(ha, szFileName); -        while(pHash != NULL) -        { -            dwLocales++; -            pHash = GetNextHashEntry(ha, pFirstHash, pHash); -        } - -        // Test if there is enough space to copy the locales -        if(*pdwMaxLocales < dwLocales) -        { -            *pdwMaxLocales = dwLocales; -            return ERROR_INSUFFICIENT_BUFFER; -        } - -        // Enum the locales -        pFirstHash = pHash = GetFirstHashEntry(ha, szFileName); -        while(pHash != NULL) -        { -            *plcLocales++ = pHash->lcLocale; -            pHash = GetNextHashEntry(ha, pFirstHash, pHash); -        } -    } -    else -    { -        // There must be space for 1 locale -        if(*pdwMaxLocales < 1) -        { -            *pdwMaxLocales = 1; -            return ERROR_INSUFFICIENT_BUFFER; -        } - -        // For nameless access, always return 1 locale -        pFileEntry = GetFileEntryByIndex(ha, dwFileIndex); -        pHash = ha->pHashTable + pFileEntry->dwHashIndex; -        *plcLocales = pHash->lcLocale; -        dwLocales = 1; -    } - -    // Give the caller the total number of found locales -    *pdwMaxLocales = dwLocales; -    return ERROR_SUCCESS; -} - -//----------------------------------------------------------------------------- -// SFileHasFile -// -//   hMpq          - Handle of opened MPQ archive -//   szFileName    - Name of file to look for - -bool WINAPI SFileHasFile(HANDLE hMpq, const char * szFileName) -{ -    TMPQArchive * ha = (TMPQArchive *)hMpq; -    TFileEntry * pFileEntry; -    DWORD dwFlagsToCheck = MPQ_FILE_EXISTS; -    DWORD dwFileIndex = 0; -    char szPrefixBuffer[MAX_PATH]; -    bool bIsPseudoName; -    int nError = ERROR_SUCCESS; - -    if(!IsValidMpqHandle(hMpq)) -        nError = ERROR_INVALID_HANDLE; -    if(szFileName == NULL || *szFileName == 0) -        nError = ERROR_INVALID_PARAMETER; - -    // Prepare the file opening -    if(nError == ERROR_SUCCESS) +    // Parse all files with that name +    pFirstHash = pHash = GetFirstHashEntry(ha, szFileName); +    while(pHash != NULL)      { -        // Different processing for pseudo-names -        bIsPseudoName = IsPseudoFileName(szFileName, &dwFileIndex); - -        // Walk through the MPQ and all patches -        while(ha != NULL) -        { -            // Verify presence of the file -            pFileEntry = (bIsPseudoName == false) ? GetFileEntryLocale(ha, GetPatchFileName(ha, szFileName, szPrefixBuffer), lcFileLocale) -                                                  : GetFileEntryByIndex(ha, dwFileIndex); -            // Verify the file flags -            if(pFileEntry != NULL && (pFileEntry->dwFlags & dwFlagsToCheck) == MPQ_FILE_EXISTS) -                return true; - -            // If this is patched archive, go to the patch -            dwFlagsToCheck = MPQ_FILE_EXISTS | MPQ_FILE_PATCH_FILE; -            ha = ha->haPatch; -        } +        // Put the locales to the buffer +        if(PtrLocales != NULL && dwLocales < dwMaxLocales) +            *PtrLocales++ = pHash->lcLocale; +        dwLocales++; -        // Not found, sorry -        nError = ERROR_FILE_NOT_FOUND; +        // Get the next locale +        pHash = GetNextHashEntry(ha, pFirstHash, pHash);      } -    // Cleanup -    SetLastError(nError); -    return false; +    // Give the caller the number of locales and return +    PtrMaxLocales[0] = dwLocales; +    return (dwLocales <= dwMaxLocales) ? ERROR_SUCCESS : ERROR_INSUFFICIENT_BUFFER;  } -  //-----------------------------------------------------------------------------  // SFileOpenFileEx  //  //   hMpq          - Handle of opened MPQ archive  //   szFileName    - Name of file to open  //   dwSearchScope - Where to search -//   phFile        - Pointer to store opened file handle +//   PtrFile       - Pointer to store opened file handle -bool WINAPI SFileOpenFileEx(HANDLE hMpq, const char * szFileName, DWORD dwSearchScope, HANDLE * phFile) +bool WINAPI SFileOpenFileEx(HANDLE hMpq, const char * szFileName, DWORD dwSearchScope, HANDLE * PtrFile)  { -    TMPQArchive * ha = (TMPQArchive *)hMpq; +    TMPQArchive * ha = IsValidMpqHandle(hMpq);      TFileEntry  * pFileEntry = NULL;      TMPQFile    * hf = NULL; +    DWORD dwHashIndex = HASH_ENTRY_FREE;      DWORD dwFileIndex = 0;      bool bOpenByIndex = false;      int nError = ERROR_SUCCESS;      // Don't accept NULL pointer to file handle -    if(phFile == NULL) +    if(szFileName == NULL || *szFileName == 0) +        nError = ERROR_INVALID_PARAMETER; + +    // When opening a file from MPQ, the handle must be valid +    if(dwSearchScope != SFILE_OPEN_LOCAL_FILE && ha == NULL) +        nError = ERROR_INVALID_HANDLE; + +    // When not checking for existence, the pointer to file handle must be valid +    if(dwSearchScope != SFILE_OPEN_CHECK_EXISTS && PtrFile == NULL)          nError = ERROR_INVALID_PARAMETER;      // Prepare the file opening @@ -297,40 +257,28 @@ bool WINAPI SFileOpenFileEx(HANDLE hMpq, const char * szFileName, DWORD dwSearch          {              case SFILE_OPEN_FROM_MPQ:              case SFILE_OPEN_BASE_FILE: +            case SFILE_OPEN_CHECK_EXISTS: -                if(!IsValidMpqHandle(hMpq)) -                { -                    nError = ERROR_INVALID_HANDLE; -                    break; -                } - -                if(szFileName == NULL || *szFileName == 0) -                { -                    nError = ERROR_INVALID_PARAMETER; -                    break; -                } -                  // Check the pseudo-file name -                if(IsPseudoFileName(szFileName, &dwFileIndex)) +                if((bOpenByIndex = IsPseudoFileName(szFileName, &dwFileIndex)) == true)                  { -                    pFileEntry = GetFileEntryByIndex(ha, dwFileIndex); -                    bOpenByIndex = true; -                    if(pFileEntry == NULL) -                        nError = ERROR_FILE_NOT_FOUND; +                    // Get the file entry for the file +                    if(dwFileIndex > ha->dwFileTableSize) +                        break; +                    pFileEntry = ha->pFileTable + dwFileIndex;                  }                  else                  { -                    // If this MPQ is a patched archive, open the file as patched +                    // If this MPQ has no patches, open the file from this MPQ directly                      if(ha->haPatch == NULL || dwSearchScope == SFILE_OPEN_BASE_FILE)                      { -                        // Otherwise, open the file from *this* MPQ -                        pFileEntry = GetFileEntryLocale(ha, szFileName, lcFileLocale); -                        if(pFileEntry == NULL) -                            nError = ERROR_FILE_NOT_FOUND; +                        pFileEntry = GetFileEntryLocale2(ha, szFileName, lcFileLocale, &dwHashIndex);                      } + +                    // If this MPQ is a patched archive, open the file as patched                      else                      { -                        return OpenPatchedFile(hMpq, szFileName, phFile); +                        return OpenPatchedFile(hMpq, szFileName, PtrFile);                      }                  }                  break; @@ -340,20 +288,13 @@ bool WINAPI SFileOpenFileEx(HANDLE hMpq, const char * szFileName, DWORD dwSearch                  // This open option is reserved for opening MPQ internal listfile.                  // No argument validation. Tries to open file with neutral locale first,                  // then any other available. -                pFileEntry = GetFileEntryAny(ha, szFileName); -                if(pFileEntry == NULL) -                    nError = ERROR_FILE_NOT_FOUND; +                pFileEntry = GetFileEntryLocale2(ha, szFileName, 0, &dwHashIndex);                  break;              case SFILE_OPEN_LOCAL_FILE: -                if(szFileName == NULL || *szFileName == 0) -                { -                    nError = ERROR_INVALID_PARAMETER; -                    break; -                } - -                return OpenLocalFile(szFileName, phFile);  +                // Open a local file +                return OpenLocalFile(szFileName, PtrFile);               default: @@ -361,73 +302,76 @@ bool WINAPI SFileOpenFileEx(HANDLE hMpq, const char * szFileName, DWORD dwSearch                  nError = ERROR_INVALID_PARAMETER;                  break;          } - -        // Quick return if something failed -        if(nError != ERROR_SUCCESS) -        { -            SetLastError(nError); -            *phFile = NULL; -            return false; -        }      } -    // Test if the file was not already deleted. +    // Check whether the file really exists in the MPQ      if(nError == ERROR_SUCCESS)      { -        if((pFileEntry->dwFlags & MPQ_FILE_EXISTS) == 0) +        if(pFileEntry == NULL || (pFileEntry->dwFlags & MPQ_FILE_EXISTS) == 0)              nError = ERROR_FILE_NOT_FOUND; -        if(pFileEntry->dwFlags & ~MPQ_FILE_VALID_FLAGS) +        if(pFileEntry != NULL && pFileEntry->dwFlags & ~MPQ_FILE_VALID_FLAGS)              nError = ERROR_NOT_SUPPORTED;      } -    // Allocate file handle -    if(nError == ERROR_SUCCESS) +    // Did the caller just wanted to know if the file exists? +    if(nError == ERROR_SUCCESS && dwSearchScope != SFILE_OPEN_CHECK_EXISTS)      { +        // Allocate file handle          hf = CreateFileHandle(ha, pFileEntry); -        if(hf == NULL) -            nError = ERROR_NOT_ENOUGH_MEMORY; -    } - -    // Initialize file handle -    if(nError == ERROR_SUCCESS) -    { -        // If the MPQ has sector CRC enabled, enable if for the file -        if(ha->dwFlags & MPQ_FLAG_CHECK_SECTOR_CRC) -            hf->bCheckSectorCRCs = true; - -        // If we know the real file name, copy it to the file entry -        if(bOpenByIndex == false) +        if(hf != NULL)          { -            // If there is no file name yet, allocate it -            AllocateFileName(ha, pFileEntry, szFileName); - -            // If the file is encrypted, we should detect the file key -            if(pFileEntry->dwFlags & MPQ_FILE_ENCRYPTED) +            // Get the hash index for the file +            if(ha->pHashTable != NULL && dwHashIndex == HASH_ENTRY_FREE) +                dwHashIndex = FindHashIndex(ha, dwFileIndex); +            if(dwHashIndex != HASH_ENTRY_FREE) +                hf->pHashEntry = ha->pHashTable + dwHashIndex; +            hf->dwHashIndex = dwHashIndex; + +            // If the MPQ has sector CRC enabled, enable if for the file +            if(ha->dwFlags & MPQ_FLAG_CHECK_SECTOR_CRC) +                hf->bCheckSectorCRCs = true; + +            // If we know the real file name, copy it to the file entry +            if(bOpenByIndex == false)              { -                hf->dwFileKey = DecryptFileKey(szFileName, -                                               pFileEntry->ByteOffset, -                                               pFileEntry->dwFileSize, -                                               pFileEntry->dwFlags); +                // If there is no file name yet, allocate it +                AllocateFileName(ha, pFileEntry, szFileName); + +                // If the file is encrypted, we should detect the file key +                if(pFileEntry->dwFlags & MPQ_FILE_ENCRYPTED) +                { +                    hf->dwFileKey = DecryptFileKey(szFileName, +                                                   pFileEntry->ByteOffset, +                                                   pFileEntry->dwFileSize, +                                                   pFileEntry->dwFlags); +                }              }          }          else          { -            // Try to auto-detect the file name -            if(!SFileGetFileName(hf, NULL)) -                nError = GetLastError(); +            nError = ERROR_NOT_ENOUGH_MEMORY;          }      } -    // Cleanup and exit +    // Give the file entry +    if(PtrFile != NULL) +        PtrFile[0] = hf; + +    // Return error code      if(nError != ERROR_SUCCESS) -    {          SetLastError(nError); -        FreeFileHandle(hf); -        return false; -    } +    return (nError == ERROR_SUCCESS); +} -    *phFile = hf; -    return true; +//----------------------------------------------------------------------------- +// SFileHasFile +// +//   hMpq          - Handle of opened MPQ archive +//   szFileName    - Name of file to look for + +bool WINAPI SFileHasFile(HANDLE hMpq, const char * szFileName) +{ +    return SFileOpenFileEx(hMpq, szFileName, SFILE_OPEN_CHECK_EXISTS, NULL);  }  //----------------------------------------------------------------------------- diff --git a/src/SFilePatchArchives.cpp b/src/SFilePatchArchives.cpp index 21446d8..f4867b0 100644 --- a/src/SFilePatchArchives.cpp +++ b/src/SFilePatchArchives.cpp @@ -19,6 +19,31 @@  #define PATCH_SIGNATURE_MD5    0x5f35444d  #define PATCH_SIGNATURE_XFRM   0x4d524658 +#define SIZE_OF_XFRM_HEADER  0x0C + +// Header for incremental patch files +typedef struct _MPQ_PATCH_HEADER +{ +    //-- PATCH header ----------------------------------- +    DWORD dwSignature;                          // 'PTCH' +    DWORD dwSizeOfPatchData;                    // Size of the entire patch (decompressed) +    DWORD dwSizeBeforePatch;                    // Size of the file before patch +    DWORD dwSizeAfterPatch;                     // Size of file after patch +     +    //-- MD5 block -------------------------------------- +    DWORD dwMD5;                                // 'MD5_' +    DWORD dwMd5BlockSize;                       // Size of the MD5 block, including the signature and size itself +    BYTE md5_before_patch[0x10];                // MD5 of the original (unpached) file +    BYTE md5_after_patch[0x10];                 // MD5 of the patched file + +    //-- XFRM block ------------------------------------- +    DWORD dwXFRM;                               // 'XFRM' +    DWORD dwXfrmBlockSize;                      // Size of the XFRM block, includes XFRM header and patch data +    DWORD dwPatchType;                          // Type of patch ('BSD0' or 'COPY') + +    // Followed by the patch data +} MPQ_PATCH_HEADER, *PMPQ_PATCH_HEADER; +  typedef struct _BLIZZARD_BSDIFF40_FILE  {      ULONGLONG Signature; @@ -27,31 +52,61 @@ typedef struct _BLIZZARD_BSDIFF40_FILE      ULONGLONG NewFileSize;  } BLIZZARD_BSDIFF40_FILE, *PBLIZZARD_BSDIFF40_FILE; +typedef struct _BSDIFF_CTRL_BLOCK +{ +    DWORD dwAddDataLength; +    DWORD dwMovDataLength; +    DWORD dwOldMoveLength; + +} BSDIFF_CTRL_BLOCK, *PBSDIFF_CTRL_BLOCK; + +typedef struct _LOCALIZED_MPQ_INFO +{ +    const char * szNameTemplate;            // Name template +    size_t nLangOffset;                     // Offset of the language +    size_t nLength;                         // Length of the name template +} LOCALIZED_MPQ_INFO, *PLOCALIZED_MPQ_INFO; +  //-----------------------------------------------------------------------------  // Local variables -static const char * LanguageList[] = +// 4-byte groups for all languages +static const char * LanguageList = "baseteenenUSenGBenCNenTWdeDEesESesMXfrFRitITkoKRptBRptPTruRUzhCNzhTW"; + +// List of localized MPQs for World of Warcraft +static LOCALIZED_MPQ_INFO LocaleMpqs_WoW[] =  { -    "deDE", -    "enCN", -    "enGB", -    "enTW", -    "enUS", -    "esES", -    "esMX", -    "frFR", -    "koKR", -    "ptBR", -    "ptPT", -    "ruRU", -    "zhCN", -    "zhTW", -    NULL +    {"expansion1-locale-####", 18, 22},  +    {"expansion1-speech-####", 18, 22},  +    {"expansion2-locale-####", 18, 22},  +    {"expansion2-speech-####", 18, 22},  +    {"expansion3-locale-####", 18, 22},  +    {"expansion3-speech-####", 18, 22},  +    {"locale-####",             7, 11},  +    {"speech-####",             7, 11},  +    {NULL, 0, 0}  };  //-----------------------------------------------------------------------------  // Local functions +static inline bool IsPatchMetadataFile(TFileEntry * pFileEntry) +{ +    // The file must ave a name +    if(pFileEntry->szFileName != NULL && (pFileEntry->dwFlags & MPQ_FILE_PATCH_FILE) == 0) +    { +        // The file must be small +        if(0 < pFileEntry->dwFileSize && pFileEntry->dwFileSize < 0x40) +        { +            // Compare the plain name +            return (_stricmp(GetPlainFileName(pFileEntry->szFileName), PATCH_METADATA_NAME) == 0); +        } +    } + +    // Not a patch_metadata +    return false; +} +  static void Decompress_RLE(LPBYTE pbDecompressed, DWORD cbDecompressed, LPBYTE pbCompressed, DWORD cbCompressed)  {      LPBYTE pbDecompressedEnd = pbDecompressed + cbDecompressed; @@ -89,122 +144,94 @@ static void Decompress_RLE(LPBYTE pbDecompressed, DWORD cbDecompressed, LPBYTE p      }  } -static int LoadFilePatch_COPY(TMPQFile * hf, TPatchHeader * pPatchHeader) +static int LoadFilePatch_COPY(TMPQFile * hf, PMPQ_PATCH_HEADER pFullPatch)  { -    int nError = ERROR_SUCCESS; - -    // Allocate space for patch header and compressed data -    hf->pPatchHeader = (TPatchHeader *)STORM_ALLOC(BYTE, pPatchHeader->dwSizeOfPatchData); -    if(hf->pPatchHeader == NULL) -        nError = ERROR_NOT_ENOUGH_MEMORY; +    DWORD cbBytesToRead = pFullPatch->dwSizeOfPatchData - sizeof(MPQ_PATCH_HEADER); +    DWORD cbBytesRead = 0; -    // Load the patch data and decide if they are compressed or not -    if(nError == ERROR_SUCCESS) -    { -        LPBYTE pbPatchFile = (LPBYTE)hf->pPatchHeader; - -        // Copy the patch header itself -        memcpy(pbPatchFile, pPatchHeader, sizeof(TPatchHeader)); -        pbPatchFile += sizeof(TPatchHeader); - -        // Load the rest of the patch -        if(!SFileReadFile((HANDLE)hf, pbPatchFile, pPatchHeader->dwSizeOfPatchData - sizeof(TPatchHeader), NULL, NULL)) -            nError = GetLastError(); -    } - -    return nError; +    // Simply load the rest of the patch +    SFileReadFile((HANDLE)hf, (pFullPatch + 1), cbBytesToRead, &cbBytesRead, NULL); +    return (cbBytesRead == cbBytesToRead) ? ERROR_SUCCESS : ERROR_FILE_CORRUPT;  } -static int LoadFilePatch_BSD0(TMPQFile * hf, TPatchHeader * pPatchHeader) +static int LoadFilePatch_BSD0(TMPQFile * hf, PMPQ_PATCH_HEADER pFullPatch)  { -    LPBYTE pbDecompressed = NULL; +    LPBYTE pbDecompressed = (LPBYTE)(pFullPatch + 1);      LPBYTE pbCompressed = NULL;      DWORD cbDecompressed = 0;      DWORD cbCompressed = 0;      DWORD dwBytesRead = 0;      int nError = ERROR_SUCCESS; -    // Allocate space for compressed data -    cbCompressed = pPatchHeader->dwXfrmBlockSize - SIZE_OF_XFRM_HEADER; -    pbCompressed = STORM_ALLOC(BYTE, cbCompressed); -    if(pbCompressed == NULL) -        nError = ERROR_NOT_ENOUGH_MEMORY; +    // Calculate the size of compressed data +    cbDecompressed = pFullPatch->dwSizeOfPatchData - sizeof(MPQ_PATCH_HEADER); +    cbCompressed = pFullPatch->dwXfrmBlockSize - SIZE_OF_XFRM_HEADER; -    // Read the compressed patch data -    if(nError == ERROR_SUCCESS) -    { -        // Load the rest of the header -        SFileReadFile((HANDLE)hf, pbCompressed, cbCompressed, &dwBytesRead, NULL); -        if(dwBytesRead != cbCompressed) -            nError = ERROR_FILE_CORRUPT; -    } - -    // Get the uncompressed size of the patch -    if(nError == ERROR_SUCCESS) +    // Is that file compressed? +    if(cbCompressed < cbDecompressed)      { -        cbDecompressed = pPatchHeader->dwSizeOfPatchData - sizeof(TPatchHeader); -        hf->pPatchHeader = (TPatchHeader *)STORM_ALLOC(BYTE, pPatchHeader->dwSizeOfPatchData); -        if(hf->pPatchHeader == NULL) +        pbCompressed = STORM_ALLOC(BYTE, cbCompressed); +        if(pbCompressed == NULL)              nError = ERROR_NOT_ENOUGH_MEMORY; -    } - -    // Now decompress the patch data -    if(nError == ERROR_SUCCESS) -    { -        // Copy the patch header -        memcpy(hf->pPatchHeader, pPatchHeader, sizeof(TPatchHeader)); -        pbDecompressed = (LPBYTE)hf->pPatchHeader + sizeof(TPatchHeader); -        // Uncompress or copy the patch data -        if(cbCompressed < cbDecompressed) +        // Read the compressed patch data +        if(nError == ERROR_SUCCESS)          { -            Decompress_RLE(pbDecompressed, cbDecompressed, pbCompressed, cbCompressed); -        } -        else -        { -            assert(cbCompressed == cbDecompressed); -            memcpy(pbDecompressed, pbCompressed, cbCompressed); +            SFileReadFile((HANDLE)hf, pbCompressed, cbCompressed, &dwBytesRead, NULL); +            if(dwBytesRead != cbCompressed) +                nError = ERROR_FILE_CORRUPT;          } + +        // Decompress the data +        if(nError == ERROR_SUCCESS) +            Decompress_RLE(pbDecompressed, cbDecompressed, pbCompressed, cbCompressed); + +        if(pbCompressed != NULL) +            STORM_FREE(pbCompressed); +    } +    else +    { +        SFileReadFile((HANDLE)hf, pbDecompressed, cbDecompressed, &dwBytesRead, NULL); +        if(dwBytesRead != cbDecompressed) +            nError = ERROR_FILE_CORRUPT;      } -    // Free buffers and exit -    if(pbCompressed != NULL) -        STORM_FREE(pbCompressed);      return nError;  }  static int ApplyFilePatch_COPY( -    TMPQFile * hfFrom, -    TMPQFile * hf, -    TPatchHeader * pPatchHeader) +    TMPQPatcher * pPatcher, +    PMPQ_PATCH_HEADER pFullPatch, +    LPBYTE pbTarget, +    LPBYTE pbSource)  {      // Sanity checks -    assert(hf->cbFileData == (pPatchHeader->dwXfrmBlockSize - SIZE_OF_XFRM_HEADER)); -    assert(hf->pbFileData != NULL); -    hfFrom = hfFrom; +    assert(pPatcher->cbMaxFileData >= pPatcher->cbFileData); +    pFullPatch = pFullPatch;      // Copy the patch data as-is -    memcpy(hf->pbFileData, (LPBYTE)pPatchHeader + sizeof(TPatchHeader), hf->cbFileData); +    memcpy(pbTarget, pbSource, pPatcher->cbFileData);      return ERROR_SUCCESS;  }  static int ApplyFilePatch_BSD0( -    TMPQFile * hfFrom, -    TMPQFile * hf, -    TPatchHeader * pPatchHeader) +    TMPQPatcher * pPatcher, +    PMPQ_PATCH_HEADER pFullPatch, +    LPBYTE pbTarget, +    LPBYTE pbSource)  {      PBLIZZARD_BSDIFF40_FILE pBsdiff; -    LPDWORD pCtrlBlock; -    LPBYTE pbPatchData = (LPBYTE)pPatchHeader + sizeof(TPatchHeader); +    PBSDIFF_CTRL_BLOCK pCtrlBlock; +    LPBYTE pbPatchData = (LPBYTE)(pFullPatch + 1);      LPBYTE pDataBlock;      LPBYTE pExtraBlock; -    LPBYTE pbOldData = hfFrom->pbFileData; -    LPBYTE pbNewData = hf->pbFileData; +    LPBYTE pbOldData = pbSource; +    LPBYTE pbNewData = pbTarget;      DWORD dwCombineSize;      DWORD dwNewOffset = 0;                          // Current position to patch      DWORD dwOldOffset = 0;                          // Current source position      DWORD dwNewSize;                                // Patched file size -    DWORD dwOldSize = hfFrom->cbFileData;           // File size before patch +    DWORD dwOldSize = pPatcher->cbFileData;         // File size before patch      // Get pointer to the patch header      // Format of BSDIFF header corresponds to original BSDIFF, which is: @@ -221,7 +248,7 @@ static int ApplyFilePatch_BSD0(      // 0000   4 bytes   Length to copy from the BSDIFF data block the new file      // 0004   4 bytes   Length to copy from the BSDIFF extra block      // 0008   4 bytes   Size to increment source file offset -    pCtrlBlock = (LPDWORD)pbPatchData; +    pCtrlBlock = (PBSDIFF_CTRL_BLOCK)pbPatchData;      pbPatchData += (size_t)BSWAP_INT64_UNSIGNED(pBsdiff->CtrlBlockSize);      // Get the pointer to the data block @@ -235,9 +262,9 @@ static int ApplyFilePatch_BSD0(      // Now patch the file      while(dwNewOffset < dwNewSize)      { -        DWORD dwAddDataLength = BSWAP_INT32_UNSIGNED(pCtrlBlock[0]); -        DWORD dwMovDataLength = BSWAP_INT32_UNSIGNED(pCtrlBlock[1]); -        DWORD dwOldMoveLength = BSWAP_INT32_UNSIGNED(pCtrlBlock[2]); +        DWORD dwAddDataLength = BSWAP_INT32_UNSIGNED(pCtrlBlock->dwAddDataLength); +        DWORD dwMovDataLength = BSWAP_INT32_UNSIGNED(pCtrlBlock->dwMovDataLength); +        DWORD dwOldMoveLength = BSWAP_INT32_UNSIGNED(pCtrlBlock->dwOldMoveLength);          DWORD i;          // Sanity check @@ -272,175 +299,122 @@ static int ApplyFilePatch_BSD0(          if(dwOldMoveLength & 0x80000000)              dwOldMoveLength = 0x80000000 - dwOldMoveLength;          dwOldOffset += dwOldMoveLength; -        pCtrlBlock += 3; +        pCtrlBlock++;      } -    // Success +    // The size after patch must match +    if(dwNewOffset != pFullPatch->dwSizeAfterPatch) +        return ERROR_FILE_CORRUPT; + +    // Update the new data size +    pPatcher->cbFileData = dwNewOffset;      return ERROR_SUCCESS;  } - -static int LoadFilePatch(TMPQFile * hf) +static PMPQ_PATCH_HEADER LoadFullFilePatch(TMPQFile * hf, MPQ_PATCH_HEADER & PatchHeader)  { -    TPatchHeader PatchHeader; -    DWORD dwBytesRead; +    PMPQ_PATCH_HEADER pFullPatch;      int nError = ERROR_SUCCESS; -    // Read the patch header -    SFileReadFile((HANDLE)hf, &PatchHeader, sizeof(TPatchHeader), &dwBytesRead, NULL); -    if(dwBytesRead != sizeof(TPatchHeader)) -        nError = ERROR_FILE_CORRUPT; +    // BSWAP the entire header, if needed +    BSWAP_ARRAY32_UNSIGNED(&PatchHeader, sizeof(DWORD) * 6); +    BSWAP_ARRAY32_UNSIGNED(&PatchHeader.dwXFRM, sizeof(DWORD) * 3);      // Verify the signatures in the patch header -    if(nError == ERROR_SUCCESS) -    { -        // BSWAP the entire header, if needed -        BSWAP_ARRAY32_UNSIGNED(&PatchHeader, sizeof(DWORD) * 6); -        PatchHeader.dwXFRM          = BSWAP_INT32_UNSIGNED(PatchHeader.dwXFRM); -        PatchHeader.dwXfrmBlockSize = BSWAP_INT32_UNSIGNED(PatchHeader.dwXfrmBlockSize); -        PatchHeader.dwPatchType     = BSWAP_INT32_UNSIGNED(PatchHeader.dwPatchType); - -        if(PatchHeader.dwSignature != PATCH_SIGNATURE_HEADER || PatchHeader.dwMD5 != PATCH_SIGNATURE_MD5 || PatchHeader.dwXFRM != PATCH_SIGNATURE_XFRM) -            nError = ERROR_FILE_CORRUPT; -    } +    if(PatchHeader.dwSignature != PATCH_SIGNATURE_HEADER || PatchHeader.dwMD5 != PATCH_SIGNATURE_MD5 || PatchHeader.dwXFRM != PATCH_SIGNATURE_XFRM) +        return NULL; -    // Read the patch, depending on patch type -    if(nError == ERROR_SUCCESS) +    // Allocate space for patch header and compressed data +    pFullPatch = (PMPQ_PATCH_HEADER)STORM_ALLOC(BYTE, PatchHeader.dwSizeOfPatchData); +    if(pFullPatch != NULL)      { -        switch(PatchHeader.dwPatchType) +        // Copy the patch header +        memcpy(pFullPatch, &PatchHeader, sizeof(MPQ_PATCH_HEADER)); + +        // Read the patch, depending on patch type +        if(nError == ERROR_SUCCESS)          { -            case 0x59504f43:    // 'COPY' -                nError = LoadFilePatch_COPY(hf, &PatchHeader); -                break; +            switch(PatchHeader.dwPatchType) +            { +                case 0x59504f43:    // 'COPY' +                    nError = LoadFilePatch_COPY(hf, pFullPatch); +                    break; -            case 0x30445342:    // 'BSD0' -                nError = LoadFilePatch_BSD0(hf, &PatchHeader); -                break; +                case 0x30445342:    // 'BSD0' +                    nError = LoadFilePatch_BSD0(hf, pFullPatch); +                    break; -            default: -                nError = ERROR_FILE_CORRUPT; -                break; +                default: +                    nError = ERROR_FILE_CORRUPT; +                    break; +            } +        } + +        // If something failed, free the patch buffer +        if(nError != ERROR_SUCCESS) +        { +            STORM_FREE(pFullPatch); +            pFullPatch = NULL;          }      } -    return nError; +    // Give the result to the caller +    return pFullPatch;  }  static int ApplyFilePatch( -    TMPQFile * hfBase,          // The file in the base MPQ -    TMPQFile * hfPrev,          // The file in the previous MPQ -    TMPQFile * hf) +    TMPQPatcher * pPatcher, +    PMPQ_PATCH_HEADER pFullPatch)  { -    TPatchHeader * pPatchHeader = hf->pPatchHeader; -    TMPQFile * hfFrom = NULL; -    int nError = ERROR_SUCCESS; +    LPBYTE pbSource = (pPatcher->nCounter & 0x1) ? pPatcher->pbFileData2 : pPatcher->pbFileData1; +    LPBYTE pbTarget = (pPatcher->nCounter & 0x1) ? pPatcher->pbFileData1 : pPatcher->pbFileData2; +    int nError;      // Sanity checks -    assert(hf->pbFileData == NULL); - -    // Either take the base version or the previous version -    if(!memcmp(hfBase->FileDataMD5, pPatchHeader->md5_before_patch, MD5_DIGEST_SIZE)) -        hfFrom = hfBase; -    if(!memcmp(hfPrev->FileDataMD5, pPatchHeader->md5_before_patch, MD5_DIGEST_SIZE)) -        hfFrom = hfPrev; -    if(hfFrom == NULL) -        return ERROR_FILE_CORRUPT; - -    // Allocate the buffer for patched file content -    hf->pbFileData = STORM_ALLOC(BYTE, pPatchHeader->dwSizeAfterPatch); -    hf->cbFileData = pPatchHeader->dwSizeAfterPatch; -    if(hf->pbFileData == NULL) -        return ERROR_NOT_ENOUGH_MEMORY; +    assert(pFullPatch->dwSizeAfterPatch <= pPatcher->cbMaxFileData); -    // Apply the patch -    if(nError == ERROR_SUCCESS) +    // Apply the patch according to the type +    switch(pFullPatch->dwPatchType)      { -        switch(pPatchHeader->dwPatchType) -        { -            case 0x59504f43:    // 'COPY' -                nError = ApplyFilePatch_COPY(hfFrom, hf, pPatchHeader); -                break; +        case 0x59504f43:    // 'COPY' +            nError = ApplyFilePatch_COPY(pPatcher, pFullPatch, pbTarget, pbSource); +            break; -            case 0x30445342:    // 'BSD0' -                nError = ApplyFilePatch_BSD0(hfFrom, hf, pPatchHeader); -                break; +        case 0x30445342:    // 'BSD0' +            nError = ApplyFilePatch_BSD0(pPatcher, pFullPatch, pbTarget, pbSource); +            break; -            default: -                nError = ERROR_FILE_CORRUPT; -                break; -        } +        default: +            nError = ERROR_FILE_CORRUPT; +            break;      }      // Verify MD5 after patch -    if(nError == ERROR_SUCCESS && pPatchHeader->dwSizeAfterPatch != 0) +    if(nError == ERROR_SUCCESS && pFullPatch->dwSizeAfterPatch != 0)      {          // Verify the patched file -        if(!VerifyDataBlockHash(hf->pbFileData, hf->cbFileData, pPatchHeader->md5_after_patch)) +        if(!VerifyDataBlockHash(pbTarget, pFullPatch->dwSizeAfterPatch, pFullPatch->md5_after_patch))              nError = ERROR_FILE_CORRUPT;          // Copy the MD5 of the new block -        memcpy(hf->FileDataMD5, pPatchHeader->md5_after_patch, MD5_DIGEST_SIZE); +        memcpy(pPatcher->this_md5, pFullPatch->md5_after_patch, MD5_DIGEST_SIZE);      }      return nError;  } -static void FreePatchData(TMPQFile * hf) -{ -    STORM_FREE(hf->pbFileData); -    hf->pbFileData = NULL; -    hf->cbFileData = 0; - -    STORM_FREE(hf->pPatchHeader); -    hf->pPatchHeader = NULL; -} -  //-----------------------------------------------------------------------------  // Local functions (patch prefix matching) -static TFileEntry * FindMd5ListFile(TMPQArchive * ha) -{ -    TFileEntry * pFileEntry = ha->pFileTable + ha->dwFileTableSize; -    char * szLstName; -    size_t nTryCount = 0; -    size_t nLength; - -    // Check every file entry for "*-md5.lst". -    // Go backwards, as the entry is usually at the end of the file table -    while(pFileEntry > ha->pFileTable && nTryCount < 10) -    { -        // The file name must be valid -        if(pFileEntry->szFileName != NULL) -        { -            // Get the name and length -            szLstName = pFileEntry->szFileName; -            nLength = strlen(szLstName); - -            // Check for the tail name -            if(!_stricmp(szLstName + nLength - 8, "-md5.lst")) -                return pFileEntry; -        } - -        // Move back -        pFileEntry--; -        nTryCount++; -    } - -    // Not found, sorry -    return NULL; -} - -static bool CreatePatchPrefix(TMPQArchive * ha, const char * szFileName, const char * szPrefixEnd) +static bool CreatePatchPrefix(TMPQArchive * ha, const char * szFileName, size_t nLength)  {      TMPQNamePrefix * pNewPrefix; -    size_t nLength;      // If the end of the patch prefix was not entered, find it -    if(szFileName != NULL && szPrefixEnd == NULL) -        szPrefixEnd = szFileName + strlen(szFileName); +    if(szFileName != NULL && nLength == 0) +        nLength = strlen(szFileName);      // Create the patch prefix -    nLength = (szPrefixEnd - szFileName);      pNewPrefix = (TMPQNamePrefix *)STORM_ALLOC(BYTE, sizeof(TMPQNamePrefix) + nLength);      if(pNewPrefix != NULL)      { @@ -465,7 +439,7 @@ static bool IsMatchingPatchFile(      const char * szFileName,      LPBYTE pbFileMd5)  { -    TPatchHeader PatchHeader = {0}; +    MPQ_PATCH_HEADER PatchHeader = {0};      HANDLE hFile = NULL;      DWORD dwTransferred = 0;      bool bResult = false; @@ -474,12 +448,12 @@ static bool IsMatchingPatchFile(      if(SFileOpenFileEx((HANDLE)ha, szFileName, SFILE_OPEN_BASE_FILE, &hFile))      {          // Load the patch header -        SFileReadFile(hFile, &PatchHeader, sizeof(TPatchHeader), &dwTransferred, NULL); +        SFileReadFile(hFile, &PatchHeader, sizeof(MPQ_PATCH_HEADER), &dwTransferred, NULL);          BSWAP_ARRAY32_UNSIGNED(pPatchHeader, sizeof(DWORD) * 6);          // If the file contains an incremental patch,          // compare the "MD5 before patching" with the base file MD5 -        if(dwTransferred == sizeof(TPatchHeader) && PatchHeader.dwSignature == PATCH_SIGNATURE_HEADER) +        if(dwTransferred == sizeof(MPQ_PATCH_HEADER) && PatchHeader.dwSignature == PATCH_SIGNATURE_HEADER)              bResult = (!memcmp(PatchHeader.md5_before_patch, pbFileMd5, MD5_DIGEST_SIZE));          // Close the file @@ -489,51 +463,87 @@ static bool IsMatchingPatchFile(      return bResult;  } -static const char * GetLstFileLanguage(const char * szFileName) +static const char * FindArchiveLanguage(TMPQArchive * ha, PLOCALIZED_MPQ_INFO pMpqInfo)  { -    char szLstSuffix[0x80]; -    size_t nLength; -    size_t nSuffixLength; - -    // Each language-dependent file ends with "xxXX-md5.lst" -    nLength = strlen(szFileName); -    if(nLength < 12) -        return NULL; +    TFileEntry * pFileEntry; +    const char * szLanguage = LanguageList; +    char szFileName[0x40]; -    // Try each and every possibility -    for(size_t i = 0; LanguageList[i] != NULL; i++) +    // Iterate through all localized languages +    while(pMpqInfo->szNameTemplate != NULL)      { -        nSuffixLength = sprintf(szLstSuffix, "%s-md5.lst", LanguageList[i]); -        assert(nSuffixLength == 12); +        // Iterate through all languages +        for(szLanguage = LanguageList; szLanguage[0] != 0; szLanguage += 4) +        { +            // Construct the file name +            memcpy(szFileName, pMpqInfo->szNameTemplate, pMpqInfo->nLength); +            szFileName[pMpqInfo->nLangOffset + 0] = szLanguage[0]; +            szFileName[pMpqInfo->nLangOffset + 1] = szLanguage[1]; +            szFileName[pMpqInfo->nLangOffset + 2] = szLanguage[2]; +            szFileName[pMpqInfo->nLangOffset + 3] = szLanguage[3]; + +            // Append the suffix +            memcpy(szFileName + pMpqInfo->nLength, "-md5.lst", 9); + +            // Check whether the name exists +            pFileEntry = GetFileEntryLocale(ha, szFileName, 0); +            if(pFileEntry != NULL) +                return szLanguage; +        } -        if(!_stricmp(szFileName + nLength - nSuffixLength, szLstSuffix)) -            return LanguageList[i]; +        // Move to the next language name +        pMpqInfo++;      } +    // Not found      return NULL;  } -static bool FindPatchPrefix_WoW_13164_13623(TMPQArchive * haBase, TMPQArchive * haPatch) +static TFileEntry * FindBaseLstFile(TMPQArchive * ha)  {      TFileEntry * pFileEntry; -    const char * szFilePrefix = "Base";      const char * szLanguage; -    char szNamePrefix[0x10]; -    int nLength; +    char szFileName[0x40]; -    // Find a *-md5.lst file in the base archive -    pFileEntry = FindMd5ListFile(haBase); -    if(pFileEntry == NULL) -        return false; +    // Prepare the file name tenplate +    memcpy(szFileName, "####-md5.lst", 13); -    // Language-specific MPQs have the language identifier right before extension -    szLanguage = GetLstFileLanguage(pFileEntry->szFileName); -    if(szLanguage != NULL) -        szFilePrefix = szLanguage; +    // Try all languages +    for(szLanguage = LanguageList; szLanguage[0] != 0; szLanguage++) +    { +        // Copy the language name +        szFileName[0] = szLanguage[0]; +        szFileName[1] = szLanguage[1]; +        szFileName[2] = szLanguage[2]; +        szFileName[3] = szLanguage[3]; + +        // Check whether this file exists +        pFileEntry = GetFileEntryLocale(ha, szFileName, 0); +        if(pFileEntry != NULL) +            return pFileEntry; +    } -    // Format the name prefix -    nLength = sprintf(szNamePrefix, "%s\\", szFilePrefix); -    return CreatePatchPrefix(haPatch, szNamePrefix, &szNamePrefix[nLength]); +    return NULL; +} + +static bool FindPatchPrefix_WoW_13164_13623(TMPQArchive * haBase, TMPQArchive * haPatch) +{ +    const char * szPatchPrefix; +    char szNamePrefix[0x08]; + +    // Try to find the language of the MPQ archive +    szPatchPrefix = FindArchiveLanguage(haBase, LocaleMpqs_WoW); +    if(szPatchPrefix == NULL) +        szPatchPrefix = "Base"; + +    // Format the patch prefix +    szNamePrefix[0] = szPatchPrefix[0]; +    szNamePrefix[1] = szPatchPrefix[1]; +    szNamePrefix[2] = szPatchPrefix[2]; +    szNamePrefix[3] = szPatchPrefix[3]; +    szNamePrefix[4] = '\\'; +    szNamePrefix[5] = 0; +    return CreatePatchPrefix(haPatch, szNamePrefix, 5);  }  // @@ -555,60 +565,68 @@ static bool FindPatchPrefix_WoW_13164_13623(TMPQArchive * haBase, TMPQArchive *  // We need to match the file by its MD5  // +  static bool FindPatchPrefix_SC2(TMPQArchive * haBase, TMPQArchive * haPatch)  { -    TFileEntry * pFileTableEnd; -    TFileEntry * pFileEntry; +    TMPQNamePrefix * pPatchPrefix;      TFileEntry * pBaseEntry; -    const char * szPlainName;      char * szLstFileName; +    char * szPlainName;      size_t cchWorkBuffer = 0x400; -    size_t cchBaseName; -    size_t cchDirName;      bool bResult = false; -    // Find a *-md5.lst file in the base archive -    pBaseEntry = FindMd5ListFile(haBase); -    if(pBaseEntry == NULL) -        return false; -    cchBaseName = strlen(pBaseEntry->szFileName) + 1; - -    // Allocate working buffer for merging LST file -    szLstFileName = STORM_ALLOC(char, cchWorkBuffer); -    if(szLstFileName != NULL) +    // First-level patches: Find the same file within the patch archive +    // and verify by MD5-before-patch +    if(haBase->haPatch == NULL)      { -        // Find that file in the patch MPQ -        pFileTableEnd = haPatch->pFileTable + haPatch->dwFileTableSize; -        for(pFileEntry = haPatch->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++) +        TFileEntry * pFileTableEnd = haPatch->pFileTable + haPatch->dwFileTableSize; +        TFileEntry * pFileEntry; + +        // Allocate working buffer for merging LST file +        szLstFileName = STORM_ALLOC(char, cchWorkBuffer); +        if(szLstFileName != NULL)          { -            // Find the "(patch_metadata)" file within that folder -            // Note that the file is always relatively small and contains the patch prefix -            // Checking for file size greatly speeds up the search process -            if(pFileEntry->szFileName && !(pFileEntry->dwFlags & MPQ_FILE_PATCH_FILE) && (0 < pFileEntry->dwFileSize && pFileEntry->dwFileSize < 0x40)) -            { -                // If the plain file name matches, we need to check its MD5 -                szPlainName = GetPlainFileName(pFileEntry->szFileName); -                cchDirName = (size_t)(szPlainName - pFileEntry->szFileName); +            // Find a *-md5.lst file in the base archive +            pBaseEntry = FindBaseLstFile(haBase); +            if(pBaseEntry == NULL) +                return false; -                // The file name must not too long and must be PATCH_METADATA_NAME -                if((cchDirName + cchBaseName) < cchWorkBuffer && _stricmp(szPlainName, PATCH_METADATA_NAME) == 0) +            // Parse the entire file table +            for(pFileEntry = haPatch->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++) +            { +                // Look for "patch_metadata" file +                if(IsPatchMetadataFile(pFileEntry))                  { -                    // Construct the name of the eventuall LST file -                    memcpy(szLstFileName, pFileEntry->szFileName, cchDirName); -                    memcpy(szLstFileName + cchDirName, pBaseEntry->szFileName, cchBaseName); +                    // Construct the name of the MD5 file +                    strcpy(szLstFileName, pFileEntry->szFileName); +                    szPlainName = (char *)GetPlainFileName(szLstFileName); +                    strcpy(szPlainName, pBaseEntry->szFileName); -                    // If there is the "*-md5.lst" file in that directory, we check its MD5 +                    // Check for matching MD5 file                      if(IsMatchingPatchFile(haPatch, szLstFileName, pBaseEntry->md5))                      { -                        bResult = CreatePatchPrefix(haPatch, pFileEntry->szFileName, szPlainName); +                        bResult = CreatePatchPrefix(haPatch, szLstFileName, (size_t)(szPlainName - szLstFileName));                          break;                      }                  }              } + +            // Delete the merge buffer +            STORM_FREE(szLstFileName);          } +    } -        // Free the work buffer -        STORM_FREE(szLstFileName); +    // For second-level patches, just take the patch prefix from the lower level patch +    else +    { +        // There must be at least two patches in the chain +        assert(haBase->haPatch->pPatchPrefix != NULL); +        pPatchPrefix = haBase->haPatch->pPatchPrefix; + +        // Copy the patch prefix +        bResult = CreatePatchPrefix(haPatch, +                                    pPatchPrefix->szPatchPrefix, +                                    pPatchPrefix->nLength);      }      return bResult; @@ -618,23 +636,22 @@ static bool FindPatchPrefix(TMPQArchive * haBase, TMPQArchive * haPatch, const c  {      // If the patch prefix was explicitly entered, we use that one      if(szPatchPathPrefix != NULL) -        return CreatePatchPrefix(haPatch, szPatchPathPrefix, szPatchPathPrefix + strlen(szPatchPathPrefix)); +        return CreatePatchPrefix(haPatch, szPatchPathPrefix, 0); -    // Patches for World of Warcraft - mostly the do not use prefix. -    // Those who do, they have the (patch_metadata) file present in the "base" subdirectory. +    // Patches for World of Warcraft - they mostly do not use prefix.      // All patches that use patch prefix have the "base\\(patch_metadata) file present -    if(GetFileEntryAny(haPatch, "base\\" PATCH_METADATA_NAME)) +    if(GetFileEntryLocale(haPatch, "base\\" PATCH_METADATA_NAME, 0))          return FindPatchPrefix_WoW_13164_13623(haBase, haPatch);      // Updates for Starcraft II      // Match: LocalizedData\GameHotkeys.txt <==> Campaigns\Liberty.SC2Campaign\enGB.SC2Data\LocalizedData\GameHotkeys.txt       // All Starcraft II base archives seem to have the file "StreamingBuckets.txt" present -    if(GetFileEntryAny(haBase, "StreamingBuckets.txt")) +    if(GetFileEntryLocale(haBase, "StreamingBuckets.txt", 0))          return FindPatchPrefix_SC2(haBase, haPatch);      // Diablo III patch MPQs don't use patch prefix      // Hearthstone MPQs don't use patch prefix -    CreatePatchPrefix(haPatch, NULL, NULL); +    CreatePatchPrefix(haPatch, NULL, 0);      return true;  } @@ -643,11 +660,11 @@ static bool FindPatchPrefix(TMPQArchive * haBase, TMPQArchive * haPatch, const c  bool IsIncrementalPatchFile(const void * pvData, DWORD cbData, LPDWORD pdwPatchedFileSize)  { -    TPatchHeader * pPatchHeader = (TPatchHeader *)pvData; +    PMPQ_PATCH_HEADER pPatchHeader = (PMPQ_PATCH_HEADER)pvData;      BLIZZARD_BSDIFF40_FILE DiffFile;      DWORD dwPatchType; -    if(cbData >= sizeof(TPatchHeader) + sizeof(BLIZZARD_BSDIFF40_FILE)) +    if(cbData >= sizeof(MPQ_PATCH_HEADER) + sizeof(BLIZZARD_BSDIFF40_FILE))      {          dwPatchType = BSWAP_INT32_UNSIGNED(pPatchHeader->dwPatchType);          if(dwPatchType == 0x30445342) @@ -666,6 +683,40 @@ bool IsIncrementalPatchFile(const void * pvData, DWORD cbData, LPDWORD pdwPatche      return false;  } +int Patch_InitPatcher(TMPQPatcher * pPatcher, TMPQFile * hf) +{ +    DWORD cbMaxFileData = 0; + +    // Overflow check +    if((sizeof(MPQ_PATCH_HEADER) + cbMaxFileData) < cbMaxFileData) +        return ERROR_NOT_ENOUGH_MEMORY; +    if(hf->hfPatch == NULL) +        return ERROR_INVALID_PARAMETER; + +    // Initialize the entire structure with zeros +    memset(pPatcher, 0, sizeof(TMPQPatcher)); + +    // Copy the MD5 of the current file +    memcpy(pPatcher->this_md5, hf->pFileEntry->md5, MD5_DIGEST_SIZE); + +    // Find out the biggest data size needed during the patching process +    while(hf != NULL) +    { +        if(hf->pFileEntry->dwFileSize > cbMaxFileData) +            cbMaxFileData = hf->pFileEntry->dwFileSize; +        hf = hf->hfPatch; +    } + +    // Allocate primary and secondary buffer +    pPatcher->pbFileData1 = STORM_ALLOC(BYTE, cbMaxFileData); +    pPatcher->pbFileData2 = STORM_ALLOC(BYTE, cbMaxFileData); +    if(!pPatcher->pbFileData1 || !pPatcher->pbFileData2) +        return ERROR_NOT_ENOUGH_MEMORY; + +    pPatcher->cbMaxFileData = cbMaxFileData; +    return ERROR_SUCCESS; +} +  //  // Note: The patch may either be applied to the base file or to the previous version  // In Starcraft II, Mods\Core.SC2Mod\Base.SC2Data, file StreamingBuckets.txt: @@ -685,59 +736,101 @@ bool IsIncrementalPatchFile(const void * pvData, DWORD cbData, LPDWORD pdwPatche  // 9 patches in a row, each requiring 70 MB memory (35 MB patch data + 35 MB work buffer)  // -int PatchFileData(TMPQFile * hf) +int Patch_Process(TMPQPatcher * pPatcher, TMPQFile * hf)  { +    PMPQ_PATCH_HEADER pFullPatch; +    MPQ_PATCH_HEADER PatchHeader1; +    MPQ_PATCH_HEADER PatchHeader2 = {0};      TMPQFile * hfBase = hf; -    TMPQFile * hfPrev = hf; +    DWORD cbBytesRead = 0;      int nError = ERROR_SUCCESS; -    // We need to calculate the MD5 of the entire file -    assert(hf->pbFileData != NULL); -    assert(hf->cbFileData != 0); -    CalculateDataBlockHash(hf->pbFileData, hf->cbFileData, hf->FileDataMD5); +    // Move to the first patch +    assert(hfBase->pbFileData == NULL); +    assert(hfBase->cbFileData == 0); +    hf = hf->hfPatch; + +    // Read the header of the current patch +    SFileReadFile((HANDLE)hf, &PatchHeader1, sizeof(MPQ_PATCH_HEADER), &cbBytesRead, NULL); +    if(cbBytesRead != sizeof(MPQ_PATCH_HEADER)) +        return ERROR_FILE_CORRUPT; -    // Apply all patches -    for(hf = hf->hfPatch; hf != NULL; hf = hf->hfPatch) +    // Perform the patching process +    while(nError == ERROR_SUCCESS && hf != NULL)      { -        // This must be true -        assert(hf->pFileEntry->dwFlags & MPQ_FILE_PATCH_FILE); +        // Try to read the next patch header. If the md5_before_patch +        // still matches we go directly to the next one and repeat +        while(hf->hfPatch != NULL) +        { +            // Attempt to read the patch header +            SFileReadFile((HANDLE)hf->hfPatch, &PatchHeader2, sizeof(MPQ_PATCH_HEADER), &cbBytesRead, NULL); +            if(cbBytesRead != sizeof(MPQ_PATCH_HEADER)) +                return ERROR_FILE_CORRUPT; -        // Make sure that the patch data is loaded -        nError = LoadFilePatch(hf); -        if(nError != ERROR_SUCCESS) -            break; +            // Compare the md5_before_patch +            if(memcmp(PatchHeader2.md5_before_patch, pPatcher->this_md5, MD5_DIGEST_SIZE)) +                break; -        // Apply the patch -        nError = ApplyFilePatch(hfBase, hfPrev, hf); -        if(nError != ERROR_SUCCESS) -            break; +            // Move one patch fuhrter +            PatchHeader1 = PatchHeader2; +            hf = hf->hfPatch; +        } -        // Only keep base file version and previous version -        if(hfPrev != hfBase) -            FreePatchData(hfPrev); +        // Allocate memory for the patch data +        pFullPatch = LoadFullFilePatch(hf, PatchHeader1); +        if(pFullPatch != NULL) +        { +            // Apply the patch +            nError = ApplyFilePatch(pPatcher, pFullPatch); +            STORM_FREE(pFullPatch); +        } +        else +        { +            nError = ERROR_FILE_CORRUPT; +        } -        // Is this the last patch in the chain? -        if(hf->hfPatch == NULL) -            break; -        hfPrev = hf; +        // Move to the next patch +        PatchHeader1 = PatchHeader2; +        pPatcher->nCounter++; +        hf = hf->hfPatch;      } -    // When done, we need to rewrite the base file data -    // with the last of the patch chain +    // Put the result data to the file structure      if(nError == ERROR_SUCCESS)      { -        // Free the base file data -        STORM_FREE(hfBase->pbFileData); - -        // Switch the latest patched data to the base file -        hfBase->pbFileData = hf->pbFileData; -        hfBase->cbFileData = hf->cbFileData; -        hf->pbFileData = NULL; -        hf->cbFileData = 0; +        // Swap the pointer to the file data structure +        if(pPatcher->nCounter & 0x01) +        { +            hfBase->pbFileData = pPatcher->pbFileData2; +            pPatcher->pbFileData2 = NULL; +        } +        else +        { +            hfBase->pbFileData = pPatcher->pbFileData1; +            pPatcher->pbFileData1 = NULL; +        } + +        // Also supply the data size +        hfBase->cbFileData = pPatcher->cbFileData; +    } + +    return ERROR_SUCCESS; +} + +void Patch_Finalize(TMPQPatcher * pPatcher) +{ +    if(pPatcher != NULL) +    { +        if(pPatcher->pbFileData1 != NULL) +            STORM_FREE(pPatcher->pbFileData1); +        if(pPatcher->pbFileData2 != NULL) +            STORM_FREE(pPatcher->pbFileData2); +         +        memset(pPatcher, 0, sizeof(TMPQPatcher));      } -    return nError;  } +  //-----------------------------------------------------------------------------  // Public functions diff --git a/src/SFileReadFile.cpp b/src/SFileReadFile.cpp index 2451865..3293cf6 100644 --- a/src/SFileReadFile.cpp +++ b/src/SFileReadFile.cpp @@ -563,6 +563,7 @@ static int ReadMpqFileSectorFile(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos  static int ReadMpqFilePatchFile(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos, DWORD dwToRead, LPDWORD pdwBytesRead)  { +    TMPQPatcher Patcher;      DWORD dwBytesToRead = dwToRead;      DWORD dwBytesRead = 0;      int nError = ERROR_SUCCESS; @@ -570,27 +571,26 @@ static int ReadMpqFilePatchFile(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos,      // Make sure that the patch file is loaded completely      if(nError == ERROR_SUCCESS && hf->pbFileData == NULL)      { -        // Load the original file and store its content to "pbOldData" -        hf->pbFileData = STORM_ALLOC(BYTE, hf->pFileEntry->dwFileSize); -        hf->cbFileData = hf->pFileEntry->dwFileSize; -        if(hf->pbFileData == NULL) -            return ERROR_NOT_ENOUGH_MEMORY; +        // Initialize patching process and allocate data +        nError = Patch_InitPatcher(&Patcher, hf); +        if(nError != ERROR_SUCCESS) +            return nError; -        // Read the file data +        // Set the current data size +        Patcher.cbFileData = hf->pFileEntry->dwFileSize; + +        // Initialize the patcher object with initial file data          if(hf->pFileEntry->dwFlags & MPQ_FILE_SINGLE_UNIT) -            nError = ReadMpqFileSingleUnit(hf, hf->pbFileData, 0, hf->cbFileData, &dwBytesRead); +            nError = ReadMpqFileSingleUnit(hf, Patcher.pbFileData1, 0, Patcher.cbFileData, &dwBytesRead);          else -            nError = ReadMpqFileSectorFile(hf, hf->pbFileData, 0, hf->cbFileData, &dwBytesRead); - -        // Fix error code -        if(nError == ERROR_SUCCESS && dwBytesRead != hf->cbFileData) -            nError = ERROR_FILE_CORRUPT; +            nError = ReadMpqFileSectorFile(hf, Patcher.pbFileData1, 0, Patcher.cbFileData, &dwBytesRead); -        // Patch the file data +        // Perform the patching process          if(nError == ERROR_SUCCESS) -            nError = PatchFileData(hf); +            nError = Patch_Process(&Patcher, hf); -        // Reset number of bytes read to zero +        // Finalize the patcher structure +        Patch_Finalize(&Patcher);          dwBytesRead = 0;      } diff --git a/src/SFileVerify.cpp b/src/SFileVerify.cpp index 65bad00..45ff58e 100644 --- a/src/SFileVerify.cpp +++ b/src/SFileVerify.cpp @@ -790,6 +790,7 @@ int SSignFileCreate(TMPQArchive * ha)      if(ha->dwFileFlags3 != 0)      {          // The (signature) file must be non-encrypted and non-compressed +        assert(ha->dwFlags & MPQ_FLAG_SIGNATURE_NEW);          assert(ha->dwFileFlags3 == MPQ_FILE_EXISTS);          assert(ha->dwReservedFiles > 0); @@ -809,11 +810,11 @@ int SSignFileCreate(TMPQArchive * ha)              memset(EmptySignature, 0, sizeof(EmptySignature));              nError = SFileAddFile_Write(hf, EmptySignature, (DWORD)sizeof(EmptySignature), 0);              SFileAddFile_Finish(hf); -        } -        // Clear the invalid mark -        ha->dwFlags &= ~MPQ_FLAG_SIGNATURE_INVALID; -        ha->dwReservedFiles--; +            // Clear the invalid mark +            ha->dwFlags &= ~(MPQ_FLAG_SIGNATURE_NEW | MPQ_FLAG_SIGNATURE_NONE); +            ha->dwReservedFiles--; +        }      }      return nError; @@ -1043,7 +1044,7 @@ bool WINAPI SFileSignArchive(HANDLE hMpq, DWORD dwSignatureType)      {          // Turn the signature on. The signature will          // be applied when the archive is closed -        ha->dwFlags |= MPQ_FLAG_SIGNATURE_INVALID | MPQ_FLAG_CHANGED; +        ha->dwFlags |= MPQ_FLAG_SIGNATURE_NEW | MPQ_FLAG_CHANGED;          ha->dwFileFlags3 = MPQ_FILE_EXISTS;          ha->dwReservedFiles++;      } diff --git a/src/StormCommon.h b/src/StormCommon.h index 25f148e..3363a70 100644 --- a/src/StormCommon.h +++ b/src/StormCommon.h @@ -105,19 +105,19 @@ typedef struct _MPQ_SIGNATURE_INFO  //  - Memory freeing function doesn't have to test the pointer to NULL  // -#if defined(_MSC_VER) && defined(_DEBUG) - -#define STORM_ALLOC(type, nitems)        (type *)HeapAlloc(GetProcessHeap(), 0, ((nitems) * sizeof(type))) -#define STORM_REALLOC(type, ptr, nitems) (type *)HeapReAlloc(GetProcessHeap(), 0, ptr, ((nitems) * sizeof(type))) -#define STORM_FREE(ptr)                  HeapFree(GetProcessHeap(), 0, ptr) - -#else +//#if defined(_MSC_VER) && defined(_DEBUG) +// +//#define STORM_ALLOC(type, nitems)        (type *)HeapAlloc(GetProcessHeap(), 0, ((nitems) * sizeof(type))) +//#define STORM_REALLOC(type, ptr, nitems) (type *)HeapReAlloc(GetProcessHeap(), 0, ptr, ((nitems) * sizeof(type))) +//#define STORM_FREE(ptr)                  HeapFree(GetProcessHeap(), 0, ptr) +// +//#else  #define STORM_ALLOC(type, nitems)        (type *)malloc((nitems) * sizeof(type))  #define STORM_REALLOC(type, ptr, nitems) (type *)realloc(ptr, ((nitems) * sizeof(type)))  #define STORM_FREE(ptr)                  free(ptr) -#endif +//#endif  //-----------------------------------------------------------------------------  // StormLib internal global variables @@ -181,7 +181,7 @@ int ConvertMpqHeaderToFormat4(TMPQArchive * ha, ULONGLONG MpqOffset, ULONGLONG F  TMPQHash * FindFreeHashEntry(TMPQArchive * ha, DWORD dwStartIndex, DWORD dwName1, DWORD dwName2, LCID lcLocale);  TMPQHash * GetFirstHashEntry(TMPQArchive * ha, const char * szFileName);  TMPQHash * GetNextHashEntry(TMPQArchive * ha, TMPQHash * pFirstHash, TMPQHash * pPrevHash); -TMPQHash * AllocateHashEntry(TMPQArchive * ha, TFileEntry * pFileEntry); +TMPQHash * AllocateHashEntry(TMPQArchive * ha, TFileEntry * pFileEntry, LCID lcLocale);  TMPQExtHeader * LoadExtTable(TMPQArchive * ha, ULONGLONG ByteOffset, size_t Size, DWORD dwSignature, DWORD dwKey);  TMPQHetTable * LoadHetTable(TMPQArchive * ha); @@ -197,10 +197,10 @@ int  CreateHashTable(TMPQArchive * ha, DWORD dwHashTableSize);  int  LoadAnyHashTable(TMPQArchive * ha);  int  BuildFileTable(TMPQArchive * ha);  int  DefragmentFileTable(TMPQArchive * ha); -int  ShrinkMalformedMpqTables(TMPQArchive * ha); +int  CreateFileTable(TMPQArchive * ha, DWORD dwFileTableSize);  int  RebuildHetTable(TMPQArchive * ha); -int  RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize, DWORD dwNewMaxFileCount); +int  RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize);  int  SaveMPQTables(TMPQArchive * ha);  TMPQHetTable * CreateHetTable(DWORD dwEntryCount, DWORD dwTotalCount, DWORD dwHashBitSize, LPBYTE pbSrcData); @@ -210,18 +210,17 @@ TMPQBetTable * CreateBetTable(DWORD dwMaxFileCount);  void FreeBetTable(TMPQBetTable * pBetTable);  // Functions for finding files in the file table -TFileEntry * GetFileEntryAny(TMPQArchive * ha, const char * szFileName); +TFileEntry * GetFileEntryLocale2(TMPQArchive * ha, const char * szFileName, LCID lcLocale, LPDWORD PtrHashIndex);  TFileEntry * GetFileEntryLocale(TMPQArchive * ha, const char * szFileName, LCID lcLocale); -TFileEntry * GetFileEntryExact(TMPQArchive * ha, const char * szFileName, LCID lcLocale); -TFileEntry * GetFileEntryByIndex(TMPQArchive * ha, DWORD dwIndex); +TFileEntry * GetFileEntryExact(TMPQArchive * ha, const char * szFileName, LCID lcLocale, LPDWORD PtrHashIndex);  // Allocates file name in the file entry  void AllocateFileName(TMPQArchive * ha, TFileEntry * pFileEntry, const char * szFileName);  // Allocates new file entry in the MPQ tables. Reuses existing, if possible -TFileEntry * AllocateFileEntry(TMPQArchive * ha, const char * szFileName, LCID lcLocale); -int  RenameFileEntry(TMPQArchive * ha, TFileEntry * pFileEntry, const char * szNewFileName); -void DeleteFileEntry(TMPQArchive * ha, TFileEntry * pFileEntry); +TFileEntry * AllocateFileEntry(TMPQArchive * ha, const char * szFileName, LCID lcLocale, LPDWORD PtrHashIndex); +int  RenameFileEntry(TMPQArchive * ha, TMPQFile * hf, const char * szNewFileName); +int  DeleteFileEntry(TMPQArchive * ha, TMPQFile * hf);  // Invalidates entries for (listfile) and (attributes)  void InvalidateInternalFiles(TMPQArchive * ha); @@ -257,11 +256,27 @@ int  WriteSectorChecksums(TMPQFile * hf);  int  WriteMemDataMD5(TFileStream * pStream, ULONGLONG RawDataOffs, void * pvRawData, DWORD dwRawDataSize, DWORD dwChunkSize, LPDWORD pcbTotalSize);  int  WriteMpqDataMD5(TFileStream * pStream, ULONGLONG RawDataOffs, DWORD dwRawDataSize, DWORD dwChunkSize);  void FreeFileHandle(TMPQFile *& hf); +void FreeArchiveHandle(TMPQArchive *& ha); -bool IsIncrementalPatchFile(const void * pvData, DWORD cbData, LPDWORD pdwPatchedFileSize); -int  PatchFileData(TMPQFile * hf); +//----------------------------------------------------------------------------- +// Patch functions -void FreeArchiveHandle(TMPQArchive *& ha); +// Structure used for the patching process +typedef struct _TMPQPatcher +{ +    BYTE this_md5[MD5_DIGEST_SIZE];             // MD5 of the current file state +    LPBYTE pbFileData1;                         // Primary working buffer +    LPBYTE pbFileData2;                         // Secondary working buffer +    DWORD cbMaxFileData;                        // Maximum allowed size of the patch data +    DWORD cbFileData;                           // Current size of the result data +    DWORD nCounter;                             // Counter of the patch process + +} TMPQPatcher; + +bool IsIncrementalPatchFile(const void * pvData, DWORD cbData, LPDWORD pdwPatchedFileSize); +int Patch_InitPatcher(TMPQPatcher * pPatcher, TMPQFile * hf); +int Patch_Process(TMPQPatcher * pPatcher, TMPQFile * hf); +void Patch_Finalize(TMPQPatcher * pPatcher);  //-----------------------------------------------------------------------------  // Utility functions @@ -276,7 +291,7 @@ void CopyFileName(TCHAR * szTarget, const char * szSource, size_t cchLength);  void CopyFileName(char * szTarget, const TCHAR * szSource, size_t cchLength);  //----------------------------------------------------------------------------- -// Support for adding files to the MPQ +// Internal support for MPQ modifications  int SFileAddFile_Init(      TMPQArchive * ha, @@ -320,12 +335,17 @@ int SSignFileFinish(TMPQArchive * ha);  // Dump data support  #ifdef __STORMLIB_DUMP_DATA__ +  void DumpMpqHeader(TMPQHeader * pHeader); +void DumpHashTable(TMPQHash * pHashTable, DWORD dwHashTableSize);  void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable);  #else +  #define DumpMpqHeader(h)           /* */ +#define DumpHashTable(h, s)        /* */  #define DumpHetAndBetTable(h, b)   /* */ +  #endif  #endif // __STORMCOMMON_H__ diff --git a/src/StormLib.h b/src/StormLib.h index 3d4fddf..a94d485 100644 --- a/src/StormLib.h +++ b/src/StormLib.h @@ -169,6 +169,7 @@ extern "C" {  // Values for SFileOpenFile  #define SFILE_OPEN_FROM_MPQ         0x00000000  // Open the file from the MPQ archive +#define SFILE_OPEN_CHECK_EXISTS     0xFFFFFFFC  // Only check whether the file exists  #define SFILE_OPEN_BASE_FILE        0xFFFFFFFD  // Reserved for StormLib internal use  #define SFILE_OPEN_ANY_LOCALE       0xFFFFFFFE  // Reserved for StormLib internal use  #define SFILE_OPEN_LOCAL_FILE       0xFFFFFFFF  // Open a local file @@ -180,12 +181,15 @@ extern "C" {  #define MPQ_FLAG_HASH_TABLE_CUT     0x00000008  // The hash table goes beyond EOF  #define MPQ_FLAG_BLOCK_TABLE_CUT    0x00000010  // The hash table goes beyond EOF  #define MPQ_FLAG_CHECK_SECTOR_CRC   0x00000020  // Checking sector CRC when reading files -#define MPQ_FLAG_LISTFILE_INVALID   0x00000040  // If set, it means that the (listfile) has been invalidated -#define MPQ_FLAG_ATTRIBUTES_INVALID 0x00000080  // If set, it means that the (attributes) has been invalidated -#define MPQ_FLAG_SIGNATURE_INVALID  0x00000100  // If set, it means that the (signature) has been invalidated -#define MPQ_FLAG_SAVING_TABLES      0x00000200  // If set, we are saving MPQ internal files and MPQ tables -#define MPQ_FLAG_PATCH              0x00000400  // If set, this MPQ is a patch archive -#define MPQ_FLAG_WAR3_MAP           0x00000800  // If set, this MPQ is a map for Warcraft III +#define MPQ_FLAG_SAVING_TABLES      0x00000040  // If set, we are saving MPQ internal files and MPQ tables +#define MPQ_FLAG_PATCH              0x00000080  // If set, this MPQ is a patch archive +#define MPQ_FLAG_WAR3_MAP           0x00000100  // If set, this MPQ is a map for Warcraft III +#define MPQ_FLAG_LISTFILE_NONE      0x00000200  // Set when no (listfile) was found in InvalidateInternalFiles +#define MPQ_FLAG_LISTFILE_NEW       0x00000400  // Set when (listfile) invalidated by InvalidateInternalFiles +#define MPQ_FLAG_ATTRIBUTES_NONE    0x00000800  // Set when no (attributes) was found in InvalidateInternalFiles +#define MPQ_FLAG_ATTRIBUTES_NEW     0x00001000  // Set when (attributes) invalidated by InvalidateInternalFiles +#define MPQ_FLAG_SIGNATURE_NONE     0x00002000  // Set when no (signature) was found in InvalidateInternalFiles +#define MPQ_FLAG_SIGNATURE_NEW      0x00004000  // Set when (signature) invalidated by InvalidateInternalFiles  // Values for TMPQArchive::dwSubType  #define MPQ_SUBTYPE_MPQ             0x00000000  // The file is a MPQ file (Blizzard games) @@ -650,31 +654,6 @@ typedef struct _TPatchInfo      // Followed by the sector table (variable length)  } TPatchInfo; -// Header for PTCH files  -typedef struct _TPatchHeader -{ -    //-- PATCH header ----------------------------------- -    DWORD dwSignature;                          // 'PTCH' -    DWORD dwSizeOfPatchData;                    // Size of the entire patch (decompressed) -    DWORD dwSizeBeforePatch;                    // Size of the file before patch -    DWORD dwSizeAfterPatch;                     // Size of file after patch -     -    //-- MD5 block -------------------------------------- -    DWORD dwMD5;                                // 'MD5_' -    DWORD dwMd5BlockSize;                       // Size of the MD5 block, including the signature and size itself -    BYTE md5_before_patch[0x10];                // MD5 of the original (unpached) file -    BYTE md5_after_patch[0x10];                 // MD5 of the patched file - -    //-- XFRM block ------------------------------------- -    DWORD dwXFRM;                               // 'XFRM' -    DWORD dwXfrmBlockSize;                      // Size of the XFRM block, includes XFRM header and patch data -    DWORD dwPatchType;                          // Type of patch ('BSD0' or 'COPY') - -    // Followed by the patch data -} TPatchHeader; - -#define SIZE_OF_XFRM_HEADER  0x0C -  // This is the combined file entry for maintaining file list in the MPQ.  // This structure is combined from block table, hi-block table,  // (attributes) file and from (listfile). @@ -683,14 +662,11 @@ typedef struct _TFileEntry      ULONGLONG FileNameHash;                     // Jenkins hash of the file name. Only used when the MPQ has BET table.      ULONGLONG ByteOffset;                       // Position of the file content in the MPQ, relative to the MPQ header      ULONGLONG FileTime;                         // FileTime from the (attributes) file. 0 if not present. -    DWORD     dwHashIndex;                      // Index to the hash table. Only used when the MPQ has classic hash table      DWORD     dwFileSize;                       // Decompressed size of the file      DWORD     dwCmpSize;                        // Compressed size of the file (i.e., size of the file data in the MPQ)      DWORD     dwFlags;                          // File flags (from block table) -    USHORT    lcLocale;                         // Locale ID for the file -    USHORT    wPlatform;                        // Platform ID for the file      DWORD     dwCrc32;                          // CRC32 from (attributes) file. 0 if not present. -    unsigned char md5[MD5_DIGEST_SIZE];         // File MD5 from the (attributes) file. 0 if not present. +    BYTE      md5[MD5_DIGEST_SIZE];             // File MD5 from the (attributes) file. 0 if not present.      char * szFileName;                          // File name. NULL if not known.  } TFileEntry; @@ -817,6 +793,7 @@ typedef struct _TMPQArchive      ULONGLONG      UserDataPos;                 // Position of user data (relative to the begin of the file)      ULONGLONG      MpqPos;                      // MPQ header offset (relative to the begin of the file) +    ULONGLONG      FileSize;                    // Size of the file at the moment of file open      struct _TMPQArchive * haPatch;              // Pointer to patch archive, if any      struct _TMPQArchive * haBase;               // Pointer to base ("previous version") archive, if any @@ -835,7 +812,6 @@ typedef struct _TMPQArchive      DWORD          dwHETBlockSize;      DWORD          dwBETBlockSize;      DWORD          dwMaxFileCount;              // Maximum number of files in the MPQ. Also total size of the file table. -    DWORD          dwHashTableSize;             // Size of the hash table. Different from hash table size in the header if the hash table was shrunk      DWORD          dwFileTableSize;             // Current size of the file table, e.g. index of the entry past the last occupied one      DWORD          dwReservedFiles;             // Number of entries reserved for internal MPQ files (listfile, attributes)      DWORD          dwSectorSize;                // Default size of one file sector @@ -860,22 +836,22 @@ typedef struct _TMPQFile  {      TFileStream  * pStream;                     // File stream. Only used on local files      TMPQArchive  * ha;                          // Archive handle +    TMPQHash     * pHashEntry;                  // Pointer to hash table entry, if the file was open using hash table      TFileEntry   * pFileEntry;                  // File entry for the file -    DWORD          dwFileKey;                   // Decryption key -    DWORD          dwFilePos;                   // Current file position      ULONGLONG      RawFilePos;                  // Offset in MPQ archive (relative to file begin)      ULONGLONG      MpqFilePos;                  // Offset in MPQ archive (relative to MPQ header) +    DWORD          dwHashIndex;                 // Hash table index (0xFFFFFFFF if not used) +    DWORD          dwFileKey;                   // Decryption key +    DWORD          dwFilePos;                   // Current file position      DWORD          dwMagic;                     // 'FILE'      struct _TMPQFile * hfPatch;                 // Pointer to opened patch file -    TPatchHeader * pPatchHeader;                // Patch header. Only used if the file is a patch file -    LPBYTE         pbFileData;                  // Data of the file (single unit files, patched files) -    DWORD          cbFileData;                  // Size of file data -    BYTE           FileDataMD5[MD5_DIGEST_SIZE];// MD5 hash of the loaded file data. Used during patch process      TPatchInfo   * pPatchInfo;                  // Patch info block, preceding the sector table -    DWORD        * SectorOffsets;               // Position of each file sector, relative to the begin of the file. Only for compressed files. -    DWORD        * SectorChksums;               // Array of sector checksums (either ADLER32 or MD5) values for each file sector +    LPDWORD        SectorOffsets;               // Position of each file sector, relative to the begin of the file. Only for compressed files. +    LPDWORD        SectorChksums;               // Array of sector checksums (either ADLER32 or MD5) values for each file sector +    LPBYTE         pbFileData;                  // Data of the file (single unit files, patched files) +    DWORD          cbFileData;                  // Size of file data      DWORD          dwCompression0;              // Compression that will be used on the first file sector      DWORD          dwSectorCount;               // Number of sectors in the file      DWORD          dwPatchedFileSize;           // Size of patched file. Used when saving patch file to the MPQ @@ -900,7 +876,6 @@ typedef struct _SFILE_FIND_DATA  {      char   cFileName[MAX_PATH];                 // Full name of the found file      char * szPlainName;                         // Plain name of the found file -    DWORD  dwHashIndex;                         // Hash table index for the file      DWORD  dwBlockIndex;                        // Block table index for the file      DWORD  dwFileSize;                          // File size in bytes      DWORD  dwFileFlags;                         // MPQ file flags diff --git a/src/huffman/huff.cpp b/src/huffman/huff.cpp index cf5ae05..c973580 100644 --- a/src/huffman/huff.cpp +++ b/src/huffman/huff.cpp @@ -406,10 +406,7 @@ void THTreeItem::RemoveItem()  THuffmannTree::THuffmannTree(bool bCompression)  { -    // TODO: Obsolete, delete this!! -//  InitializeHTListHead(&ItemLinks);      pFirst = pLast = LIST_HEAD(); -       MinValidValue = 1;      ItemsUsed = 0; diff --git a/test/StormTest.cpp b/test/StormTest.cpp index a986844..4a8388f 100644 --- a/test/StormTest.cpp +++ b/test/StormTest.cpp @@ -1482,7 +1482,7 @@ static TFileData * LoadMpqFile(TLogHelper * pLogger, HANDLE hMpq, const char * s      pLogger->PrintProgress("Loading file %s ...", GetShortPlainName(szFileName));  #if defined(_MSC_VER) && defined(_DEBUG) -//  if(!_stricmp(szFileName, "DragonSeaTurtle_Portrait.mdx")) +//  if(!_stricmp(szFileName, "manifest-cards.csv"))  //      DebugBreak();  #endif @@ -1619,9 +1619,7 @@ static int SearchArchive(          // Increment number of files          dwFileCount++; -//      if(!_stricmp(sf.cFileName, "DBFilesClient\\Item-Sparse.db2")) -//          DebugBreak(); -//      if(!_stricmp(sf.cFileName, "TriggerLibs\\NativeLib.galaxy")) +//      if(!_stricmp(sf.cFileName, "Interface\\Glues\\CREDITS\\1024px-Blade3_final2.blp"))  //          DebugBreak();          if(dwTestFlags & TEST_FLAG_MOST_PATCHED) @@ -2878,43 +2876,6 @@ static int TestOpenArchive_CraftedUserData(const char * szPlainName, const char      return nError;  } - -static int TestOpenArchive_CompactingTest(const char * szPlainName, const char * szListFile) -{ -    TLogHelper Logger("CompactingTest", szPlainName); -    HANDLE hMpq = NULL; -    char szFullListName[MAX_PATH]; -    int nError = ERROR_SUCCESS; - -    // Create copy of the listfile -    if(szListFile != NULL) -    { -        nError = CreateFileCopy(&Logger, szListFile, szListFile, szFullListName, 0, 0); -        szListFile = szFullListName; -    } - -    // Create copy of the archive -    if(nError == ERROR_SUCCESS) -    { -        nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szPlainName, &hMpq); -    } - -    if(nError == ERROR_SUCCESS) -    { -        // Compact the archive -        Logger.PrintProgress("Compacting archive %s ...", szPlainName); -        if(!SFileSetCompactCallback(hMpq, CompactCallback, &Logger)) -            nError = Logger.PrintError("Failed to set the compact callback"); - -        if(!SFileCompactArchive(hMpq, szListFile, false)) -            nError = Logger.PrintError("Failed to compact archive %s", szPlainName); -         -        SFileCloseArchive(hMpq); -    } - -    return nError; -} -  static int ForEachFile_VerifyFileChecksum(const char * szFullPath)  {      const char * szShortPlainName = GetShortPlainName(szFullPath); @@ -3716,7 +3677,7 @@ static int TestCreateArchive_ListFilePos(const char * szPlainName)      TLogHelper Logger("ListFilePos", szPlainName);      HANDLE hMpq = NULL;                 // Handle of created archive       char szArchivedName[MAX_PATH]; -    DWORD dwMaxFileCount = 0x1E; +    DWORD dwMaxFileCount = 0x0E;      DWORD dwFileCount = 0;      size_t i;      int nError; @@ -3724,7 +3685,7 @@ static int TestCreateArchive_ListFilePos(const char * szPlainName)      // Create a new archive with the limit of 0x20 files      nError = CreateNewArchive(&Logger, szPlainName, MPQ_CREATE_ARCHIVE_V4 | MPQ_CREATE_LISTFILE | MPQ_CREATE_ATTRIBUTES, dwMaxFileCount, &hMpq); -    // Add 0x1E files +    // Add maximum files files      if(nError == ERROR_SUCCESS)      {          for(i = 0; i < dwMaxFileCount; i++) @@ -4052,7 +4013,7 @@ int main(int argc, char * argv[])      // Open a stream, paired with remote master (takes hell lot of time!)  //  if(nError == ERROR_SUCCESS)  //      nError = TestReadFile_MasterMirror("MPQ_2013_v4_alternate-downloaded.MPQ", "http://www.zezula.net\\mpqs\\alternate.zip", false); - +*/      // Search in listfile      if(nError == ERROR_SUCCESS)          nError = TestSearchListFile("ListFile_Blizzard.txt"); @@ -4127,9 +4088,9 @@ int main(int argc, char * argv[])      if(nError == ERROR_SUCCESS)          nError = TestOpenArchive("MPQ_2015_v1_ProtectedMap_Spazy.w3x"); -    // Open an empty archive (found in WoW cache - it's just a header) +    // Open an protected map      if(nError == ERROR_SUCCESS) -        nError = TestOpenArchive("flem1.w3x"); +        nError = TestOpenArchive("MPQ_2015_v1_flem1.w3x");      // Open an Warcraft III map whose "(attributes)" file has (BlockTableSize-1) entries      if(nError == ERROR_SUCCESS) @@ -4242,9 +4203,6 @@ int main(int argc, char * argv[])      if(nError == ERROR_SUCCESS)          nError = TestOpenArchive_CraftedUserData("MPQ_2013_v4_expansion1.MPQ", "StormLibTest_CraftedMpq3_v4.mpq"); -//  if(nError == ERROR_SUCCESS) -//      nError = TestOpenArchive_CompactingTest("MPQ_2014_v1_CompactTest.w3x", "ListFile_Blizzard.txt"); -      if(nError == ERROR_SUCCESS)          nError = TestAddFile_FullTable("MPQ_2014_v1_out1.w3x", "MPQ_2014_v1_out2.w3x"); @@ -4302,7 +4260,7 @@ int main(int argc, char * argv[])      // Create an archive and fill it with files up to the max file count      if(nError == ERROR_SUCCESS)          nError = TestCreateArchive_FillArchive("StormLibTest_FileTableFull.mpq", MPQ_CREATE_ATTRIBUTES | MPQ_CREATE_LISTFILE); -*/ +      // Create an archive, and increment max file count several times      if(nError == ERROR_SUCCESS)          nError = TestCreateArchive_IncMaxFileCount("StormLibTest_IncMaxFileCount.mpq"); | 
