+ Improved key detection for archives with large sector sizes

10 files changed, 307 insertions, 257 deletions

diff --git a/src/SBaseCommon.cpp b/src/SBaseCommon.cpp
index ef753bd..be7a74d 100644
--- a/src/SBaseCommon.cpp
+++ b/src/SBaseCommon.cpp
@@ -266,175 +266,191 @@ DWORD GetDefaultSpecialFileFlags(DWORD dwFileSize, USHORT wFormatVersion)
 
 
 //-----------------------------------------------------------------------------
-// Encrypting and decrypting MPQ file data
+// Encrypting/Decrypting MPQ data block
 
-void EncryptMpqBlock(void * pvFileBlock, DWORD dwLength, DWORD dwSeed1)
+void EncryptMpqBlock(void * pvDataBlock, DWORD dwLength, DWORD dwKey1)
 {
-    LPDWORD block = (LPDWORD)pvFileBlock;
-    DWORD dwSeed2 = 0xEEEEEEEE;
-    DWORD ch;
+    LPDWORD DataBlock = (LPDWORD)pvDataBlock;
+    DWORD dwValue32;
+    DWORD dwKey2 = 0xEEEEEEEE;
 
     // Round to DWORDs
     dwLength >>= 2;
 
-    while(dwLength-- > 0)
+    // Encrypt the data block at array of DWORDs
+    for(DWORD i = 0; i < dwLength; i++)
     {
-        dwSeed2 += StormBuffer[0x400 + (dwSeed1 & 0xFF)];
-        ch     = *block;
-        *block++ = ch ^ (dwSeed1 + dwSeed2);
+        // Modify the second key
+        dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
 
-        dwSeed1  = ((~dwSeed1 << 0x15) + 0x11111111) | (dwSeed1 >> 0x0B);
-        dwSeed2  = ch + dwSeed2 + (dwSeed2 << 5) + 3;
+        dwValue32 = DataBlock[i];
+        DataBlock[i] = DataBlock[i] ^ (dwKey1 + dwKey2);
+
+        dwKey1 = ((~dwKey1 << 0x15) + 0x11111111) | (dwKey1 >> 0x0B);
+        dwKey2 = dwValue32 + dwKey2 + (dwKey2 << 5) + 3;
     }
 }
 
-void DecryptMpqBlock(void * pvFileBlock, DWORD dwLength, DWORD dwSeed1)
+void DecryptMpqBlock(void * pvDataBlock, DWORD dwLength, DWORD dwKey1)
 {
-    LPDWORD block = (LPDWORD)pvFileBlock;
-    DWORD dwSeed2 = 0xEEEEEEEE;
-    DWORD ch;
+    LPDWORD DataBlock = (LPDWORD)pvDataBlock;
+    DWORD dwValue32;
+    DWORD dwKey2 = 0xEEEEEEEE;
 
     // Round to DWORDs
     dwLength >>= 2;
 
-    while(dwLength-- > 0)
+    // Decrypt the data block at array of DWORDs
+    for(DWORD i = 0; i < dwLength; i++)
     {
-        dwSeed2 += StormBuffer[0x400 + (dwSeed1 & 0xFF)];
-        ch     = *block ^ (dwSeed1 + dwSeed2);
-
-        dwSeed1  = ((~dwSeed1 << 0x15) + 0x11111111) | (dwSeed1 >> 0x0B);
-        dwSeed2  = ch + dwSeed2 + (dwSeed2 << 5) + 3;
-        *block++ = ch;
+        // Modify the second key
+        dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
+        
+        DataBlock[i] = DataBlock[i] ^ (dwKey1 + dwKey2);
+        dwValue32 = DataBlock[i];
+
+        dwKey1 = ((~dwKey1 << 0x15) + 0x11111111) | (dwKey1 >> 0x0B);
+        dwKey2 = dwValue32 + dwKey2 + (dwKey2 << 5) + 3;
     }
 }
 
 /**
- * Functions tries to get file decryption key. The trick comes from sector
- * positions which are stored at the begin of each compressed file. We know the
- * file size, that means we know number of sectors that means we know the first
- * DWORD value in sector position. And if we know encrypted and decrypted value,
- * we can find the decryption key !!!
+ * Functions tries to get file decryption key. This comes from these facts
+ *
+ * - We know the decrypted value of the first DWORD in the encrypted data
+ * - We know the decrypted value of the second DWORD (at least aproximately)
+ * - There is only 256 variants of how the second key is modified
+ *
+ *  The first iteration of dwKey1 and dwKey2 is this:
+ *
+ *  dwKey2 = 0xEEEEEEEE + StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)]
+ *  dwDecrypted0 = DataBlock[0] ^ (dwKey1 + dwKey2);
+ *
+ *  This means:
+ *
+ *  (dwKey1 + dwKey2) = DataBlock[0] ^ dwDecrypted0;
  *
- * hf            - MPQ file handle
- * SectorOffsets - DWORD array of sector positions
- * ch            - Decrypted value of the first sector pos
  */
 
-DWORD DetectFileKeyBySectorSize(LPDWORD SectorOffsets, DWORD decrypted)
+DWORD DetectFileKeyBySectorSize(LPDWORD EncryptedData, DWORD dwSectorSize, DWORD dwDecrypted0)
 {
-    DWORD saveKey1;
-    DWORD temp = *SectorOffsets ^ decrypted;    // temp = seed1 + seed2
-    temp -= 0xEEEEEEEE;                 // temp = seed1 + StormBuffer[0x400 + (seed1 & 0xFF)]
+    DWORD dwDecrypted1Max = dwSectorSize + dwDecrypted0;
+    DWORD dwKey1PlusKey2;
+    DWORD DataBlock[2];
 
-    for(int i = 0; i < 0x100; i++)      // Try all 255 possibilities
-    {
-        DWORD seed1;
-        DWORD seed2 = 0xEEEEEEEE;
-        DWORD ch;
+    // We must have at least 2 DWORDs there to be able to decrypt something
+    if(dwSectorSize < 0x08)
+        return 0;
+
+    // Get the value of the combined encryption key
+    dwKey1PlusKey2 = (EncryptedData[0] ^ dwDecrypted0) - 0xEEEEEEEE;
 
-        // Try the first DWORD (We exactly know the value)
-        seed1  = temp - StormBuffer[0x400 + i];
-        seed2 += StormBuffer[0x400 + (seed1 & 0xFF)];
-        ch     = SectorOffsets[0] ^ (seed1 + seed2);
+    // Try all 256 combinations of dwKey1
+    for(DWORD i = 0; i < 0x100; i++)
+    {
+        DWORD dwSaveKey1;
+        DWORD dwKey1 = dwKey1PlusKey2 - StormBuffer[MPQ_HASH_KEY2_MIX + i];
+        DWORD dwKey2 = 0xEEEEEEEE;
 
-        if(ch != decrypted)
-            continue;
+        // Modify the second key and decrypt the first DWORD
+        dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
+        DataBlock[0] = EncryptedData[0] ^ (dwKey1 + dwKey2);
 
-        // Add 1 because we are decrypting sector positions
-        saveKey1 = seed1 + 1;
+        // Did we obtain the same value like dwDecrypted0?
+        if(DataBlock[0] == dwDecrypted0)
+        {
+            // Save this key value. Increment by one because
+            // we are decrypting sector offset table
+            dwSaveKey1 = dwKey1 + 1;
 
-        // If OK, continue and test the second value. We don't know exactly the value,
-        // but we know that the second one has lower 16 bits set to zero
-        // (no compressed sector is larger than 0xFFFF bytes)
-        seed1  = ((~seed1 << 0x15) + 0x11111111) | (seed1 >> 0x0B);
-        seed2  = ch + seed2 + (seed2 << 5) + 3;
+            // Rotate both keys
+            dwKey1 = ((~dwKey1 << 0x15) + 0x11111111) | (dwKey1 >> 0x0B);
+            dwKey2 = DataBlock[0] + dwKey2 + (dwKey2 << 5) + 3;
 
-        seed2 += StormBuffer[0x400 + (seed1 & 0xFF)];
-        ch     = SectorOffsets[1] ^ (seed1 + seed2);
+            // Modify the second key again and decrypt the second DWORD
+            dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
+            DataBlock[1] = EncryptedData[1] ^ (dwKey1 + dwKey2);
 
-        if((ch & 0xFFFF0000) == 0)
-            return saveKey1;
+            // Now compare the results
+            if(DataBlock[1] <= dwDecrypted1Max)
+                return dwSaveKey1;
+        }
     }
+
+    // Key not found
     return 0;
 }
 
-// Function tries to detect file encryption key. It expectes at least two uncompressed bytes
-DWORD DetectFileKeyByKnownContent(void * pvFileContent, DWORD nDwords, ...)
+// Function tries to detect file encryption key based on expected file content
+// It is the same function like before, except that we know the value of the second DWORD
+DWORD DetectFileKeyByKnownContent(void * pvEncryptedData, DWORD dwDecrypted0, DWORD dwDecrypted1)
 {
-    LPDWORD pdwContent = (LPDWORD)pvFileContent;
-    va_list argList;
-    DWORD dwDecrypted[0x10];
-    DWORD saveKey1;
-    DWORD dwTemp;
-    DWORD i, j;
-    
-    // We need at least two DWORDS to detect the file key
-    if(nDwords < 0x02 || nDwords > 0x10)
-        return 0;
-    memset(dwDecrypted, 0, sizeof(dwDecrypted));
-    
-    va_start(argList, nDwords);
-    for(i = 0; i < nDwords; i++)
-        dwDecrypted[i] = va_arg(argList, DWORD);
-    va_end(argList);
-    
-    dwTemp = (*pdwContent ^ dwDecrypted[0]) - 0xEEEEEEEE;
-    for(i = 0; i < 0x100; i++)      // Try all 256 possibilities
-    {
-        DWORD seed1;
-        DWORD seed2 = 0xEEEEEEEE;
-        DWORD ch;
+    LPDWORD EncryptedData = (LPDWORD)pvEncryptedData;
+    DWORD dwKey1PlusKey2;
+    DWORD DataBlock[2];
 
-        // Try the first DWORD
-        seed1  = dwTemp - StormBuffer[0x400 + i];
-        seed2 += StormBuffer[0x400 + (seed1 & 0xFF)];
-        ch     = pdwContent[0] ^ (seed1 + seed2);
+    // Get the value of the combined encryption key
+    dwKey1PlusKey2 = (EncryptedData[0] ^ dwDecrypted0) - 0xEEEEEEEE;
 
-        if(ch != dwDecrypted[0])
-            continue;
+    // Try all 256 combinations of dwKey1
+    for(DWORD i = 0; i < 0x100; i++)
+    {
+        DWORD dwSaveKey1;
+        DWORD dwKey1 = dwKey1PlusKey2 - StormBuffer[MPQ_HASH_KEY2_MIX + i];
+        DWORD dwKey2 = 0xEEEEEEEE;
 
-        saveKey1 = seed1;
+        // Modify the second key and decrypt the first DWORD
+        dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
+        DataBlock[0] = EncryptedData[0] ^ (dwKey1 + dwKey2);
 
-        // If OK, continue and test all bytes.
-        for(j = 1; j < nDwords; j++)
+        // Did we obtain the same value like dwDecrypted0?
+        if(DataBlock[0] == dwDecrypted0)
         {
-            seed1  = ((~seed1 << 0x15) + 0x11111111) | (seed1 >> 0x0B);
-            seed2  = ch + seed2 + (seed2 << 5) + 3;
+            // Save this key value
+            dwSaveKey1 = dwKey1;
 
-            seed2 += StormBuffer[0x400 + (seed1 & 0xFF)];
-            ch     = pdwContent[j] ^ (seed1 + seed2);
+            // Rotate both keys
+            dwKey1 = ((~dwKey1 << 0x15) + 0x11111111) | (dwKey1 >> 0x0B);
+            dwKey2 = DataBlock[0] + dwKey2 + (dwKey2 << 5) + 3;
 
-            if(ch == dwDecrypted[j] && j == nDwords - 1)
-                return saveKey1;
+            // Modify the second key again and decrypt the second DWORD
+            dwKey2 += StormBuffer[MPQ_HASH_KEY2_MIX + (dwKey1 & 0xFF)];
+            DataBlock[1] = EncryptedData[1] ^ (dwKey1 + dwKey2);
+
+            // Now compare the results
+            if(DataBlock[1] == dwDecrypted1)
+                return dwSaveKey1;
         }
     }
+
+    // Key not found
     return 0;
 }
 
-DWORD DetectFileKeyByContent(void * pvFileContent, DWORD dwFileSize)
+DWORD DetectFileKeyByContent(void * pvEncryptedData, DWORD dwSectorSize, DWORD dwFileSize)
 {
     DWORD dwFileKey;
 
     // Try to break the file encryption key as if it was a WAVE file
-    if(dwFileSize >= 0x0C)
+    if(dwSectorSize >= 0x0C)
     {
-        dwFileKey = DetectFileKeyByKnownContent(pvFileContent, 3, 0x46464952, dwFileSize - 8, 0x45564157);
+        dwFileKey = DetectFileKeyByKnownContent(pvEncryptedData, 0x46464952, dwFileSize - 8);
         if(dwFileKey != 0)
             return dwFileKey;
     }
 
     // Try to break the encryption key as if it was an EXE file
-    if(dwFileSize > 0x40)
+    if(dwSectorSize > 0x40)
     {
-        dwFileKey = DetectFileKeyByKnownContent(pvFileContent, 2, 0x00905A4D, 0x00000003);
+        dwFileKey = DetectFileKeyByKnownContent(pvEncryptedData, 0x00905A4D, 0x00000003);
         if(dwFileKey != 0)
             return dwFileKey;
     }
 
     // Try to break the encryption key as if it was a XML file
-    if(dwFileSize > 0x04)
+    if(dwSectorSize > 0x04)
     {
-        dwFileKey = DetectFileKeyByKnownContent(pvFileContent, 2, 0x6D783F3C, 0x6576206C);
+        dwFileKey = DetectFileKeyByKnownContent(pvEncryptedData, 0x6D783F3C, 0x6576206C);
         if(dwFileKey != 0)
             return dwFileKey;
     }
@@ -673,7 +689,7 @@ ULONGLONG FindFreeMpqSpace(TMPQArchive * ha)
 //-----------------------------------------------------------------------------
 // Common functions - MPQ File
 
-TMPQFile * CreateMpqFile(TMPQArchive * ha)
+TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry)
 {
     TMPQFile * hf;
 
@@ -683,9 +699,21 @@ TMPQFile * CreateMpqFile(TMPQArchive * ha)
     {
         // Fill the file structure
         memset(hf, 0, sizeof(TMPQFile));
-        hf->ha = ha;
-        hf->pStream = NULL;
         hf->dwMagic = ID_MPQ_FILE;
+        hf->pStream = NULL;
+        hf->ha = ha;
+
+        // If the called entered a file entry, we also copy informations from the file entry
+        if(ha != NULL && pFileEntry != NULL)
+        {
+            // Set the raw position and MPQ position
+            hf->RawFilePos = ha->MpqPos + pFileEntry->ByteOffset;
+            hf->MpqFilePos = pFileEntry->ByteOffset;
+
+            // Set the data size
+            hf->dwDataSize = pFileEntry->dwFileSize;
+            hf->pFileEntry = pFileEntry;
+        }
     }
 
     return hf;
@@ -863,7 +891,7 @@ __AllocateAndLoadPatchInfo:
 
     // Allocate space for patch header. Start with default size,
     // and if its size if bigger, then we reload them
-    hf->pPatchInfo = (TPatchInfo *)STORM_ALLOC(BYTE, dwLength);
+    hf->pPatchInfo = STORM_ALLOC(TPatchInfo, 1);
     if(hf->pPatchInfo == NULL)
         return ERROR_NOT_ENOUGH_MEMORY;
 
@@ -983,7 +1011,7 @@ int AllocateSectorOffsets(TMPQFile * hf, bool bLoadFromFile)
                 // If we don't know the file key, try to find it.
                 if(hf->dwFileKey == 0)
                 {
-                    hf->dwFileKey = DetectFileKeyBySectorSize(hf->SectorOffsets, dwSectorOffsLen);
+                    hf->dwFileKey = DetectFileKeyBySectorSize(hf->SectorOffsets, ha->dwSectorSize, dwSectorOffsLen);
                     if(hf->dwFileKey == 0)
                     {
                         STORM_FREE(hf->SectorOffsets);
@@ -1356,13 +1384,13 @@ int WriteMpqDataMD5(
 }
 
 // Frees the structure for MPQ file
-void FreeMPQFile(TMPQFile *& hf)
+void FreeFileHandle(TMPQFile *& hf)
 {
     if(hf != NULL)
     {
         // If we have patch file attached to this one, free it first
         if(hf->hfPatch != NULL)
-            FreeMPQFile(hf->hfPatch);
+            FreeFileHandle(hf->hfPatch);
 
         // Then free all buffers allocated in the file structure
         if(hf->pPatchHeader != NULL)
@@ -1377,20 +1405,21 @@ void FreeMPQFile(TMPQFile *& hf)
             STORM_FREE(hf->SectorChksums);
         if(hf->pbFileSector != NULL)
             STORM_FREE(hf->pbFileSector);
-        FileStream_Close(hf->pStream);
+        if(hf->pStream != NULL)
+            FileStream_Close(hf->pStream);
         STORM_FREE(hf);
         hf = NULL;
     }
 }
 
 // Frees the MPQ archive
-void FreeMPQArchive(TMPQArchive *& ha)
+void FreeArchiveHandle(TMPQArchive *& ha)
 {
     if(ha != NULL)
     {
         // First of all, free the patch archive, if any
         if(ha->haPatch != NULL)
-            FreeMPQArchive(ha->haPatch);
+            FreeArchiveHandle(ha->haPatch);
 
         // Close the file stream
         FileStream_Close(ha->pStream);
diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp
index 40f761a..9e95d3e 100644
--- a/src/SBaseFileTable.cpp
+++ b/src/SBaseFileTable.cpp
@@ -60,12 +60,23 @@ static int CompareFilePositions(const void * p1, const void * p2)
 {
     TMPQBlock * pBlock1 = *(TMPQBlock **)p1;
     TMPQBlock * pBlock2 = *(TMPQBlock **)p2;
+    DWORD dwFileEnd1;
+    DWORD dwFileEnd2;
 
+    // Compare file begins
     if(pBlock1->dwFilePos < pBlock2->dwFilePos)
         return -1;
     if(pBlock1->dwFilePos > pBlock2->dwFilePos)
         return +1;
 
+    // If the files begin at the same position, compare their ends
+    dwFileEnd1 = pBlock1->dwFilePos + pBlock1->dwCSize;
+    dwFileEnd2 = pBlock2->dwFilePos + pBlock2->dwCSize;
+    if(dwFileEnd1 < dwFileEnd2)
+        return -1;
+    if(dwFileEnd1 > dwFileEnd2)
+        return +1;
+
     return 0;
 }
 
@@ -321,54 +332,6 @@ static ULONGLONG DetermineArchiveSize_V2(
 }
 
 // This function converts the MPQ header so it always looks like version 4
-/*static ULONGLONG GetArchiveSize64(TMPQHeader * pHeader)
-{
-    ULONGLONG ArchiveSize = pHeader->dwHeaderSize;
-    ULONGLONG ByteOffset = pHeader->dwHeaderSize;
-
-    // If there is HET table
-    if(pHeader->HetTablePos64 != 0)
-    {
-        ByteOffset = pHeader->HetTablePos64 + pHeader->HetTableSize64;
-        if(ByteOffset > ArchiveSize)
-            ArchiveSize = ByteOffset;
-    }
-
-    // If there is BET table
-    if(pHeader->BetTablePos64 != 0)
-    {
-        ByteOffset = pHeader->BetTablePos64 + pHeader->BetTableSize64;
-        if(ByteOffset > ArchiveSize)
-            ArchiveSize = ByteOffset;
-    }
-
-    // If there is hash table
-    if(pHeader->dwHashTablePos || pHeader->wHashTablePosHi)
-    {
-        ByteOffset = MAKE_OFFSET64(pHeader->wHashTablePosHi, pHeader->dwHashTablePos) + pHeader->HashTableSize64;
-        if(ByteOffset > ArchiveSize)
-            ArchiveSize = ByteOffset;
-    }
-
-    // If there is block table
-    if(pHeader->dwBlockTablePos || pHeader->wBlockTablePosHi)
-    {
-        ByteOffset = MAKE_OFFSET64(pHeader->wBlockTablePosHi, pHeader->dwBlockTablePos) + pHeader->BlockTableSize64;
-        if(ByteOffset > ArchiveSize)
-            ArchiveSize = ByteOffset;
-    }
-
-    // If there is hi-block table
-    if(pHeader->HiBlockTablePos64)
-    {
-        ByteOffset = pHeader->HiBlockTablePos64 + pHeader->HiBlockTableSize64;
-        if(ByteOffset > ArchiveSize)
-            ArchiveSize = ByteOffset;
-    }
-
-    return ArchiveSize;
-}
-*/
 int ConvertMpqHeaderToFormat4(
     TMPQArchive * ha,
     ULONGLONG MpqOffset,
@@ -2246,7 +2209,7 @@ static void FixCompressedFileSize(
                 SortTable[nElements++] = pBlock;
         }
 
-        // Have we found at least one compressed
+        // Have we found at least one block?
         if(nElements > 0)
         {
             // Sort the table
diff --git a/src/SFileAddFile.cpp b/src/SFileAddFile.cpp
index a7d99ea..040ed57 100644
--- a/src/SFileAddFile.cpp
+++ b/src/SFileAddFile.cpp
@@ -394,7 +394,7 @@ int SFileAddFile_Init(
         lcLocale = 0;
 
     // Allocate the TMPQFile entry for newly added file
-    hf = CreateMpqFile(ha);
+    hf = CreateFileHandle(ha, NULL);
     if(hf == NULL)
         nError = ERROR_NOT_ENOUGH_MEMORY;
 
@@ -664,7 +664,7 @@ int SFileAddFile_Finish(TMPQFile * hf)
     }
 
     // Clear the add file callback
-    FreeMPQFile(hf);
+    FreeFileHandle(hf);
     return nError;
 }
 
@@ -1143,12 +1143,10 @@ bool WINAPI SFileRenameFile(HANDLE hMpq, const char * szFileName, const char * s
         // with the new decryption key
         if(pFileEntry->dwFlags & MPQ_FILE_ENCRYPTED)
         {
-            hf = CreateMpqFile(ha);
+            hf = CreateFileHandle(ha, pFileEntry);
             if(hf != NULL)
             {
                 // Recrypt the file data in the MPQ
-                hf->pFileEntry = pFileEntry;
-                hf->dwDataSize = pFileEntry->dwFileSize;
                 nError = RecryptFileData(ha, hf, szFileName, szNewFileName);
                 
                 // Update the MD5
@@ -1161,7 +1159,7 @@ bool WINAPI SFileRenameFile(HANDLE hMpq, const char * szFileName, const char * s
                                     ha->pHeader->dwRawChunkSize);
                 }
                 
-                FreeMPQFile(hf);
+                FreeFileHandle(hf);
             }
             else
             {
diff --git a/src/SFileCompactArchive.cpp b/src/SFileCompactArchive.cpp
index e36d507..3b2ba83 100644
--- a/src/SFileCompactArchive.cpp
+++ b/src/SFileCompactArchive.cpp
@@ -18,7 +18,36 @@
 /* Local functions                                                           */
 /*****************************************************************************/
 
-static int CheckIfAllFilesKnown(TMPQArchive * ha, const char * szListFile, LPDWORD pFileKeys)
+static int CheckIfAllFilesKnown(TMPQArchive * ha)
+{
+    TFileEntry * pFileTableEnd;
+    TFileEntry * pFileEntry;
+    DWORD dwBlockIndex = 0;
+    int nError = ERROR_SUCCESS;
+
+    // Verify the file table
+    if(nError == ERROR_SUCCESS)
+    {
+        pFileTableEnd = ha->pFileTable + ha->dwFileTableSize;
+        for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++, dwBlockIndex++)
+        {
+            // If there is an existing entry in the file table, check its name
+            if(pFileEntry->dwFlags & MPQ_FILE_EXISTS)
+            {
+                // The name must be valid and must not be a pseudo-name
+                if(pFileEntry->szFileName == NULL || IsPseudoFileName(pFileEntry->szFileName, NULL))
+                {
+                    nError = ERROR_UNKNOWN_FILE_NAMES;
+                    break;
+                }
+            }
+        }
+    }
+
+    return nError;
+}
+
+static int CheckIfAllKeysKnown(TMPQArchive * ha, const char * szListFile, LPDWORD pFileKeys)
 {
     TFileEntry * pFileTableEnd;
     TFileEntry * pFileEntry;
@@ -41,30 +70,49 @@ static int CheckIfAllFilesKnown(TMPQArchive * ha, const char * szListFile, LPDWO
         pFileTableEnd = ha->pFileTable + ha->dwFileTableSize;
         for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++, dwBlockIndex++)
         {
+            // If the file exists and it's encrypted
             if(pFileEntry->dwFlags & MPQ_FILE_EXISTS)
             {
+                // If we know the name, we decrypt the file key from the file name
                 if(pFileEntry->szFileName != NULL && !IsPseudoFileName(pFileEntry->szFileName, NULL))
                 {
-                    DWORD dwFileKey = 0;
+                    // Give the key to the caller
+                    pFileKeys[dwBlockIndex] = DecryptFileKey(pFileEntry->szFileName,
+                                                             pFileEntry->ByteOffset,
+                                                             pFileEntry->dwFileSize,
+                                                             pFileEntry->dwFlags);
+                    continue;
+                }
+/*
+                // If the file has a nonzero size, we can try to read few bytes of data
+                // and force to detect the decryption key that way
+                if(pFileEntry->dwFileSize > 0x10)
+                {
+                    TMPQFile * hf = NULL;
+                    DWORD dwBytesRead = 0;
+                    DWORD FileData[4];
 
-                    // Resolve the file key. Use plain file name for it
-                    if(pFileEntry->dwFlags & MPQ_FILE_ENCRYPTED)
+                    // Create file handle where we load the sector offset table
+                    hf = CreateFileHandle(ha, pFileEntry);
+                    if(hf != NULL)
                     {
-                        dwFileKey = DecryptFileKey(pFileEntry->szFileName,
-                                                   pFileEntry->ByteOffset,
-                                                   pFileEntry->dwFileSize,
-                                                   pFileEntry->dwFlags);
+                        // Call one dummy load of the first 4 bytes.
+                        // This enforces loading all buffers and also detecting of the decryption key
+                        SFileReadFile((HANDLE)hf, FileData, sizeof(FileData), &dwBytesRead, NULL);
+                        pFileKeys[dwBlockIndex] = hf->dwFileKey;
+                        FreeFileHandle(hf);
                     }
 
-                    // Give the key to the caller
-                    if(pFileKeys != NULL)
-                        pFileKeys[dwBlockIndex] = dwFileKey;
-                }
-                else
-                {
-                    nError = ERROR_UNKNOWN_FILE_NAMES;
-                    break;
+                    // If we succeeded in reading 16 bytes from the file,
+                    // we also know the encryption key
+                    if(dwBytesRead == sizeof(FileData))
+                        continue;
                 }
+*/
+                // We don't know the encryption key of this file,
+                // thus we cannot compact the file
+                nError = ERROR_UNKNOWN_FILE_NAMES;
+                break;
             }
         }
     }
@@ -373,20 +421,12 @@ static int CopyMpqFiles(TMPQArchive * ha, LPDWORD pFileKeys, TFileStream * pNewS
         if((pFileEntry->dwFlags & MPQ_FILE_EXISTS) && pFileEntry->dwFileSize != 0)
         {
             // Allocate structure for the MPQ file
-            hf = CreateMpqFile(ha);
+            hf = CreateFileHandle(ha, pFileEntry);
             if(hf == NULL)
                 return ERROR_NOT_ENOUGH_MEMORY;
 
-            // Store file entry
-            hf->pFileEntry = pFileEntry;
-
-            // Set the raw file position
-            hf->MpqFilePos = pFileEntry->ByteOffset;
-            hf->RawFilePos = ha->MpqPos + hf->MpqFilePos;
-
             // Set the file decryption key
             hf->dwFileKey = pFileKeys[pFileEntry - ha->pFileTable];
-            hf->dwDataSize = pFileEntry->dwFileSize;
 
             // If the file is a patch file, load the patch header
             if(pFileEntry->dwFlags & MPQ_FILE_PATCH_FILE)
@@ -420,13 +460,13 @@ static int CopyMpqFiles(TMPQArchive * ha, LPDWORD pFileKeys, TFileStream * pNewS
                 break;
 
             // Free buffers. This also sets "hf" to NULL.
-            FreeMPQFile(hf);
+            FreeFileHandle(hf);
         }
     }
 
     // Cleanup and exit
     if(hf != NULL)
-        FreeMPQFile(hf);
+        FreeFileHandle(hf);
     return nError;
 }
 
@@ -492,7 +532,7 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR
         // Initialize the progress variables for compact callback
         FileStream_GetSize(ha->pStream, &(ha->CompactTotalBytes));
         ha->CompactBytesProcessed = 0;
-        nError = CheckIfAllFilesKnown(ha, szListFile, pFileKeys);
+        nError = CheckIfAllKeysKnown(ha, szListFile, pFileKeys);
     }
 
     // Get the temporary file name and create it
@@ -636,7 +676,7 @@ bool WINAPI SFileSetMaxFileCount(HANDLE hMpq, DWORD dwMaxFileCount)
     // to rebuild hash table
     if(nError == ERROR_SUCCESS)
     {
-        nError = CheckIfAllFilesKnown(ha, NULL, NULL);
+        nError = CheckIfAllFilesKnown(ha);
     }
 
     // If the MPQ has a hash table, then we relocate the hash table
diff --git a/src/SFileCreateArchive.cpp b/src/SFileCreateArchive.cpp
index 5f375c8..a922e90 100644
--- a/src/SFileCreateArchive.cpp
+++ b/src/SFileCreateArchive.cpp
@@ -252,7 +252,7 @@ bool WINAPI SFileCreateArchive2(const TCHAR * szMpqName, PSFILE_CREATE_MPQ pCrea
     if(nError != ERROR_SUCCESS)
     {
         FileStream_Close(pStream);
-        FreeMPQArchive(ha);
+        FreeArchiveHandle(ha);
         SetLastError(nError);
         ha = NULL;
     }
diff --git a/src/SFileOpenArchive.cpp b/src/SFileOpenArchive.cpp
index ceb4ce1..0ca51c0 100644
--- a/src/SFileOpenArchive.cpp
+++ b/src/SFileOpenArchive.cpp
@@ -413,7 +413,7 @@ bool WINAPI SFileOpenArchive(
     if(nError != ERROR_SUCCESS)
     {
         FileStream_Close(pStream);
-        FreeMPQArchive(ha);
+        FreeArchiveHandle(ha);
         SetLastError(nError);
         ha = NULL;
     }
@@ -509,11 +509,16 @@ bool WINAPI SFileCloseArchive(HANDLE hMpq)
     TMPQArchive * ha = (TMPQArchive *)hMpq;
     bool bResult;
 
+    // Invalidate the add file callback so it won't be called
+    // when saving (listfile) and (attributes)
+    ha->pfnAddFileCB = NULL;
+    ha->pvAddFileUserData = NULL;
+
     // Flush all unsaved data to the storage
     bResult = SFileFlushArchive(hMpq);
 
     // Free all memory used by MPQ archive
-    FreeMPQArchive(ha);
+    FreeArchiveHandle(ha);
     return bResult;
 }
 
diff --git a/src/SFileOpenFileEx.cpp b/src/SFileOpenFileEx.cpp
index 31e235e..e994414 100644
--- a/src/SFileOpenFileEx.cpp
+++ b/src/SFileOpenFileEx.cpp
@@ -50,7 +50,7 @@ static bool OpenLocalFile(const char * szFileName, HANDLE * phFile)
     if(pStream != NULL)
     {
         // Allocate and initialize file handle
-        hf = CreateMpqFile(NULL);
+        hf = CreateFileHandle(NULL, NULL);
         if(hf != NULL)
         {
             hf->pStream = pStream;
@@ -381,22 +381,14 @@ bool WINAPI SFileOpenFileEx(HANDLE hMpq, const char * szFileName, DWORD dwSearch
     // Allocate file handle
     if(nError == ERROR_SUCCESS)
     {
-        if((hf = STORM_ALLOC(TMPQFile, 1)) == NULL)
+        hf = CreateFileHandle(ha, pFileEntry);
+        if(hf == NULL)
             nError = ERROR_NOT_ENOUGH_MEMORY;
     }
 
     // Initialize file handle
     if(nError == ERROR_SUCCESS)
     {
-        memset(hf, 0, sizeof(TMPQFile));
-        hf->pFileEntry = pFileEntry;
-        hf->dwMagic = ID_MPQ_FILE;
-        hf->ha = ha;
-
-        hf->MpqFilePos   = pFileEntry->ByteOffset;
-        hf->RawFilePos   = ha->MpqPos + hf->MpqFilePos;
-        hf->dwDataSize   = pFileEntry->dwFileSize;
-
         // If the MPQ has sector CRC enabled, enable if for the file
         if(ha->dwFlags & MPQ_FLAG_CHECK_SECTOR_CRC)
             hf->bCheckSectorCRCs = true;
@@ -424,18 +416,11 @@ bool WINAPI SFileOpenFileEx(HANDLE hMpq, const char * szFileName, DWORD dwSearch
         }
     }
 
-    // If the file is actually a patch file, we have to load the patch file header
-    if(nError == ERROR_SUCCESS && pFileEntry->dwFlags & MPQ_FILE_PATCH_FILE)
-    {
-        assert(hf->pPatchInfo == NULL);
-        nError = AllocatePatchInfo(hf, true);
-    }
-
     // Cleanup and exit
     if(nError != ERROR_SUCCESS)
     {
         SetLastError(nError);
-        FreeMPQFile(hf);
+        FreeFileHandle(hf);
         return false;
     }
 
@@ -457,6 +442,6 @@ bool WINAPI SFileCloseFile(HANDLE hFile)
     }
 
     // Free the structure
-    FreeMPQFile(hf);
+    FreeFileHandle(hf);
     return true;
 }
diff --git a/src/SFileReadFile.cpp b/src/SFileReadFile.cpp
index 1e1ae56..3d6a2ee 100644
--- a/src/SFileReadFile.cpp
+++ b/src/SFileReadFile.cpp
@@ -125,7 +125,7 @@ static int ReadMpqSectors(TMPQFile * hf, LPBYTE pbBuffer, DWORD dwByteOffset, DW
             // If we don't know the key, try to detect it by file content
             if(hf->dwFileKey == 0)
             {
-                hf->dwFileKey = DetectFileKeyByContent(pbInSector, dwBytesInThisSector);
+                hf->dwFileKey = DetectFileKeyByContent(pbInSector, dwBytesInThisSector, hf->dwDataSize);
                 if(hf->dwFileKey == 0)
                 {
                     nError = ERROR_UNKNOWN_FILE_KEY;
@@ -565,7 +565,7 @@ static int ReadMpqFilePatchFile(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos,
     int nError = ERROR_SUCCESS;
 
     // Make sure that the patch file is loaded completely
-    if(hf->pbFileData == NULL)
+    if(nError == ERROR_SUCCESS && hf->pbFileData == NULL)
     {
         // Load the original file and store its content to "pbOldData"
         hf->pbFileData = STORM_ALLOC(BYTE, hf->pFileEntry->dwFileSize);
@@ -672,6 +672,17 @@ bool WINAPI SFileReadFile(HANDLE hFile, void * pvBuffer, DWORD dwToRead, LPDWORD
         return false;
     }
 
+    // If we didn't load the patch info yet, do it now
+    if(hf->pFileEntry != NULL && (hf->pFileEntry->dwFlags & MPQ_FILE_PATCH_FILE) && hf->pPatchInfo == NULL)
+    {
+        nError = AllocatePatchInfo(hf, true);
+        if(nError != ERROR_SUCCESS)
+        {
+            SetLastError(nError);
+            return false;
+        }
+    }
+
     // If the file is local file, read the data directly from the stream
     if(hf->pStream != NULL)
     {
diff --git a/src/StormCommon.h b/src/StormCommon.h
index 2b2cf1d..52a5620 100644
--- a/src/StormCommon.h
+++ b/src/StormCommon.h
@@ -134,6 +134,7 @@ extern unsigned char AsciiToUpperTable[256];
 #define MPQ_HASH_NAME_A         0x100
 #define MPQ_HASH_NAME_B         0x200
 #define MPQ_HASH_FILE_KEY       0x300
+#define MPQ_HASH_KEY2_MIX       0x400
 
 DWORD HashString(const char * szFileName, DWORD dwHashType);
 DWORD HashStringSlash(const char * szFileName, DWORD dwHashType);
@@ -148,11 +149,11 @@ ULONGLONG HashStringJenkins(const char * szFileName);
 
 DWORD GetDefaultSpecialFileFlags(DWORD dwFileSize, USHORT wFormatVersion);
 
-void  EncryptMpqBlock(void * pvFileBlock, DWORD dwLength, DWORD dwKey);
-void  DecryptMpqBlock(void * pvFileBlock, DWORD dwLength, DWORD dwKey);
+void  EncryptMpqBlock(void * pvDataBlock, DWORD dwLength, DWORD dwKey);
+void  DecryptMpqBlock(void * pvDataBlock, DWORD dwLength, DWORD dwKey);
 
-DWORD DetectFileKeyBySectorSize(LPDWORD SectorOffsets, DWORD decrypted);
-DWORD DetectFileKeyByContent(void * pvFileContent, DWORD dwFileSize);
+DWORD DetectFileKeyBySectorSize(LPDWORD EncryptedData, DWORD dwSectorSize, DWORD dwSectorOffsLen);
+DWORD DetectFileKeyByContent(void * pvEncryptedData, DWORD dwSectorSize, DWORD dwFileSize);
 DWORD DecryptFileKey(const char * szFileName, ULONGLONG MpqPos, DWORD dwFileSize, DWORD dwFlags);
 
 bool IsValidMD5(LPBYTE pbMd5);
@@ -235,7 +236,7 @@ int SCompDecompressMpk(void * pvOutBuffer, int * pcbOutBuffer, void * pvInBuffer
 //-----------------------------------------------------------------------------
 // Common functions - MPQ File
 
-TMPQFile * CreateMpqFile(TMPQArchive * ha);
+TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry);
 void * LoadMpqTable(TMPQArchive * ha, ULONGLONG ByteOffset, DWORD dwCompressedSize, DWORD dwRealSize, DWORD dwKey);
 int  AllocateSectorBuffer(TMPQFile * hf);
 int  AllocatePatchInfo(TMPQFile * hf, bool bLoadFromFile);
@@ -247,12 +248,12 @@ int  WriteSectorOffsets(TMPQFile * hf);
 int  WriteSectorChecksums(TMPQFile * hf);
 int  WriteMemDataMD5(TFileStream * pStream, ULONGLONG RawDataOffs, void * pvRawData, DWORD dwRawDataSize, DWORD dwChunkSize, LPDWORD pcbTotalSize);
 int  WriteMpqDataMD5(TFileStream * pStream, ULONGLONG RawDataOffs, DWORD dwRawDataSize, DWORD dwChunkSize);
-void FreeMPQFile(TMPQFile *& hf);
+void FreeFileHandle(TMPQFile *& hf);
 
 bool IsIncrementalPatchFile(const void * pvData, DWORD cbData, LPDWORD pdwPatchedFileSize);
 int  PatchFileData(TMPQFile * hf);
 
-void FreeMPQArchive(TMPQArchive *& ha);
+void FreeArchiveHandle(TMPQArchive *& ha);
 
 //-----------------------------------------------------------------------------
 // Utility functions
diff --git a/test/Test.cpp b/test/Test.cpp
index 80230d2..96fa4f2 100644
--- a/test/Test.cpp
+++ b/test/Test.cpp
@@ -1470,11 +1470,17 @@ static TFileData * LoadMpqFile(TLogHelper * pLogger, HANDLE hMpq, const char * s
             nError = pLogger->PrintError("Failed to read the content of the file %s", szFileName);
     }
 
+    // If failed, free the buffer
+    if(nError != ERROR_SUCCESS)
+    {
+        STORM_FREE(pFileData);
+        SetLastError(nError);
+        pFileData = NULL;
+    }
+
     // Close the file and return what we got
     if(hFile != NULL)
         SFileCloseFile(hFile);
-    if(nError != ERROR_SUCCESS)
-        SetLastError(nError);
     return pFileData;
 }
 
@@ -1544,6 +1550,9 @@ static int SearchArchive(
         // Increment number of files
         dwFileCount++;
 
+//      if(!_stricmp(sf.cFileName, "OldWorld\\world\\maps\\Northrend\\Northrend.tex"))
+//          DebugBreak();
+
         if(dwTestFlags & TEST_FLAG_MOST_PATCHED)
         {
             // Load the patch count
@@ -1562,26 +1571,23 @@ static int SearchArchive(
         {
             // Load the entire file to the MPQ
             pFileData = LoadMpqFile(pLogger, hMpq, sf.cFileName);
-            if(pFileData == NULL)
+            if(pFileData != NULL)
             {
-                nError = pLogger->PrintError("Failed to load the file %s", sf.cFileName);
-                break;
-            }
+                // Hash the file data, if needed
+                if((dwTestFlags & TEST_FLAG_HASH_FILES) && !IsInternalMpqFileName(sf.cFileName))
+                    md5_process(&md5state, pFileData->FileData, pFileData->dwFileSize);
 
-            // Hash the file data, if needed
-            if((dwTestFlags & TEST_FLAG_HASH_FILES) && !IsInternalMpqFileName(sf.cFileName))
-                md5_process(&md5state, pFileData->FileData, pFileData->dwFileSize);
-
-            // Play sound files, if required
-            if((dwTestFlags & TEST_FLAG_PLAY_WAVES) && strstr(sf.cFileName, ".wav") != NULL)
-            {
+                // Play sound files, if required
+                if((dwTestFlags & TEST_FLAG_PLAY_WAVES) && strstr(sf.cFileName, ".wav") != NULL)
+                {
 #ifdef _MSC_VER
-                pLogger->PrintProgress("Playing sound %s", sf.cFileName);
-                PlaySound((LPCTSTR)pFileData->FileData, NULL, SND_MEMORY);
+                    pLogger->PrintProgress("Playing sound %s", sf.cFileName);
+                    PlaySound((LPCTSTR)pFileData->FileData, NULL, SND_MEMORY);
 #endif
-            }
+                }
 
-            STORM_FREE(pFileData);
+                STORM_FREE(pFileData);
+            }
         }
 
         bFound = SFileFindNextFile(hFind, &sf);
@@ -2627,14 +2633,26 @@ static int TestOpenArchive_CraftedUserData(const char * szPlainName, const char
 }
 
 
-static int TestOpenArchive_CompactingTest(const char * szPlainName)
+static int TestOpenArchive_CompactingTest(const char * szPlainName, const char * szListFile)
 {
     TLogHelper Logger("CompactingTest", szPlainName);
     HANDLE hMpq = NULL;
-    int nError;
+    char szFullListName[MAX_PATH];
+    int nError = ERROR_SUCCESS;
+
+    // Create copy of the listfile
+    if(szListFile != NULL)
+    {
+        nError = CreateFileCopy(&Logger, szListFile, szListFile, szFullListName, 0, 0);
+        szListFile = szFullListName;
+    }
+
+    // Create copy of the archive
+    if(nError == ERROR_SUCCESS)
+    {
+        nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szPlainName, &hMpq);
+    }
 
-    // Create copy of the archive, with interleaving some user data
-    nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szPlainName, &hMpq);
     if(nError == ERROR_SUCCESS)
     {
         // Compact the archive
@@ -2642,7 +2660,7 @@ static int TestOpenArchive_CompactingTest(const char * szPlainName)
         if(!SFileSetCompactCallback(hMpq, CompactCallback, &Logger))
             nError = Logger.PrintError("Failed to set the compact callback");
 
-        if(!SFileCompactArchive(hMpq, NULL, false))
+        if(!SFileCompactArchive(hMpq, szListFile, false))
             nError = Logger.PrintError("Failed to compact archive %s", szPlainName);
         
         SFileCloseArchive(hMpq);
@@ -3457,10 +3475,10 @@ int main(int argc, char * argv[])
     // Not a test, but rather a tool for creating links to duplicated files
 //  if(nError == ERROR_SUCCESS)
 //      nError = FindFilePairs(ForEachFile_CreateArchiveLink, "2004 - WoW\\06080", "2004 - WoW\\06299");
-/*
+
     // Search all testing archives and verify their SHA1 hash
-    if(nError == ERROR_SUCCESS)
-        nError = FindFiles(ForEachFile_VerifyFileChecksum, szMpqSubDir);
+//  if(nError == ERROR_SUCCESS)
+//      nError = FindFiles(ForEachFile_VerifyFileChecksum, szMpqSubDir);
 
     // Test reading linear file without bitmap
     if(nError == ERROR_SUCCESS)
@@ -3565,7 +3583,7 @@ int main(int argc, char * argv[])
     // Open an Warcraft III map locked by the Spazzler protector
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_Spazzler.w3x");
-*/
+
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2014_v1_ProtectedMap_Spazzler2.w3x");
 
@@ -3657,7 +3675,7 @@ int main(int argc, char * argv[])
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive_CraftedUserData("MPQ_2010_v3_expansion-locale-frFR.MPQ", "StormLibTest_CraftedMpq1_v3.mpq");
 
-    // Open a MPQ (add custom user data to it
+    // Open a MPQ (add custom user data to it)
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive_CraftedUserData("MPQ_2013_v4_SC2_EmptyMap.SC2Map", "StormLibTest_CraftedMpq2_v4.mpq");
 
@@ -3666,7 +3684,7 @@ int main(int argc, char * argv[])
         nError = TestOpenArchive_CraftedUserData("MPQ_2013_v4_expansion1.MPQ", "StormLibTest_CraftedMpq3_v4.mpq");
 
 //  if(nError == ERROR_SUCCESS)
-//      nError = TestOpenArchive_CompactingTest("MPQ_2014_v1_MapToCompact.w3x");
+//      nError = TestOpenArchive_CompactingTest("MPQ_2014_v1_CompactTest.w3x", "ListFile_Blizzard.txt");
 
     // Test modifying file with no (listfile) and no (attributes)
     if(nError == ERROR_SUCCESS)