+ Fixed possible stack overflow in HashStringJenkins

+ StormLib now creates a listfile even for empty archives to make sure it will be maintained at later point when files are added to it
+ Bug fixes in listfile loading

11 files changed, 431 insertions, 204 deletions

diff --git a/StormLib_test.vcproj b/StormLib_test.vcproj
index 09cef44..7bb6ea3 100644
--- a/StormLib_test.vcproj
+++ b/StormLib_test.vcproj
@@ -51,7 +51,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="./src/libtomcrypt/src/headers"
-				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;__STORMLIB_TEST__"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;__STORMLIB_TEST__;__STORMLIB_DUMP_DATA__"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="1"
diff --git a/src/SBaseCommon.cpp b/src/SBaseCommon.cpp
index b4d62e0..22b1221 100644
--- a/src/SBaseCommon.cpp
+++ b/src/SBaseCommon.cpp
@@ -226,7 +226,7 @@ ULONGLONG HashStringJenkins(const char * szFileName)
 {
     LPBYTE pbFileName = (LPBYTE)szFileName;
     char * szTemp;
-    char szLocFileName[0x108];
+    char szNameBuff[0x108];
     size_t nLength = 0;
     unsigned int primary_hash = 1;
     unsigned int secondary_hash = 2;
@@ -234,21 +234,22 @@ ULONGLONG HashStringJenkins(const char * szFileName)
     // Normalize the file name - convert to uppercase, and convert "/" to "\\".
     if(pbFileName != NULL)
     {
-        szTemp = szLocFileName;
-        while(*pbFileName != 0)
-            *szTemp++ = (char)AsciiToLowerTable[*pbFileName++];
-        *szTemp = 0;
+        char * szNamePtr = szNameBuff;
+        char * szNameEnd = szNamePtr + sizeof(szNameBuff);
 
-        nLength = szTemp - szLocFileName;
+        // Normalize the file name. Doesn't have to be zero terminated for hashing
+        while(szNamePtr < szNameEnd && pbFileName[0] != 0)
+            *szNamePtr++ = (char)AsciiToLowerTable[*pbFileName++];
+        nLength = szNamePtr - szNameBuff;
     }
 
     // Thanks Quantam for finding out what the algorithm is.
     // I am really getting old for reversing large chunks of assembly
     // that does hashing :-)
-    hashlittle2(szLocFileName, nLength, &secondary_hash, &primary_hash);
+    hashlittle2(szNameBuff, nLength, &secondary_hash, &primary_hash);
 
     // Combine those 2 together
-    return (ULONGLONG)primary_hash * (ULONGLONG)0x100000000ULL + (ULONGLONG)secondary_hash;
+    return ((ULONGLONG)primary_hash << 0x20) | (ULONGLONG)secondary_hash;
 }
 
 //-----------------------------------------------------------------------------
@@ -722,6 +723,43 @@ TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry)
     return hf;
 }
 
+TMPQFile * CreateWritableHandle(TMPQArchive * ha, DWORD dwFileSize)
+{
+    ULONGLONG FreeMpqSpace;
+    ULONGLONG TempPos;
+    TMPQFile * hf;
+
+    // We need to find the position in the MPQ where we save the file data
+    FreeMpqSpace = FindFreeMpqSpace(ha);
+
+    // When format V1, the size of the archive cannot exceed 4 GB
+    if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1)
+    {
+        TempPos  = FreeMpqSpace +
+                   dwFileSize +
+                  (ha->pHeader->dwHashTableSize * sizeof(TMPQHash)) + 
+                  (ha->dwFileTableSize * sizeof(TMPQBlock));
+        if((TempPos >> 32) != 0)
+        {
+            SetLastError(ERROR_DISK_FULL);
+            return NULL;
+        }
+    }
+
+    // Allocate the file handle
+    hf = CreateFileHandle(ha, NULL);
+    if(hf == NULL)
+    {
+        SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+        return NULL;
+    }
+
+    // We need to find the position in the MPQ where we save the file data
+    hf->MpqFilePos = FreeMpqSpace;
+    hf->bIsWriteHandle = true;
+    return hf;
+}
+
 // Loads a table from MPQ.
 // Can be used for hash table, block table, sector offset table or sector checksum table
 void * LoadMpqTable(
diff --git a/src/SBaseDumpData.cpp b/src/SBaseDumpData.cpp
index d2b1206..d156030 100644
--- a/src/SBaseDumpData.cpp
+++ b/src/SBaseDumpData.cpp
@@ -161,4 +161,29 @@ void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable)
     printf("-----------------------------------------------------------------------------------------\n");
 }
 
+void DumpFileTable(TFileEntry * pFileTable, DWORD dwFileTableSize)
+{
+    DWORD i;
+
+    if(pFileTable == NULL || dwFileTableSize == 0)
+        return;
+
+    printf("== File Table =================================\n");
+    for(i = 0; i < dwFileTableSize; i++, pFileTable++)
+    {
+        printf("[%04u] %08X-%08X %08X-%08X %08X-%08X 0x%08X 0x%08X 0x%08X %s\n", i,
+                        (DWORD)(pFileTable->FileNameHash >> 0x20),
+                        (DWORD)(pFileTable->FileNameHash & 0xFFFFFFFF),
+                        (DWORD)(pFileTable->ByteOffset >> 0x20),
+                        (DWORD)(pFileTable->ByteOffset & 0xFFFFFFFF),
+                        (DWORD)(pFileTable->FileTime >> 0x20),
+                        (DWORD)(pFileTable->FileTime & 0xFFFFFFFF),
+                                pFileTable->dwFileSize,
+                                pFileTable->dwCmpSize,
+                                pFileTable->dwFlags,
+                                pFileTable->szFileName != NULL ? pFileTable->szFileName : "");
+    }
+    printf("-----------------------------------------------\n\n");
+}
+
 #endif  // __STORMLIB_DUMP_DATA__
diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp
index 3e98ebb..5d7d973 100644
--- a/src/SBaseFileTable.cpp
+++ b/src/SBaseFileTable.cpp
@@ -377,6 +377,11 @@ int ConvertMpqHeaderToFormat4(
             if(pHeader->dwBlockTablePos <= pHeader->dwHeaderSize || (pHeader->dwBlockTablePos & 0x80000000))
                 ha->dwFlags |= MPQ_FLAG_MALFORMED;
 
+            // Only low byte of sector size is really used
+            if(pHeader->wSectorSize & 0xFF00)
+                ha->dwFlags |= MPQ_FLAG_MALFORMED;
+            pHeader->wSectorSize = pHeader->wSectorSize & 0xFF;
+
             // Fill the rest of the header
             memset((LPBYTE)pHeader + MPQ_HEADER_SIZE_V1, 0, sizeof(TMPQHeader) - MPQ_HEADER_SIZE_V1);
             pHeader->BlockTableSize64 = pHeader->dwBlockTableSize * sizeof(TMPQBlock);
@@ -569,6 +574,13 @@ int ConvertMpqHeaderToFormat4(
 // Support for hash table
 
 // Hash entry verification when the file table does not exist yet
+bool IsValidHashEntry(TMPQArchive * ha, TMPQHash * pHash)
+{
+    TFileEntry * pFileEntry = ha->pFileTable + pHash->dwBlockIndex;
+    return ((pHash->dwBlockIndex < ha->dwFileTableSize) && (pFileEntry->dwFlags & MPQ_FILE_EXISTS)) ? true : false;
+}
+
+// Hash entry verification when the file table does not exist yet
 static bool IsValidHashEntry1(TMPQArchive * ha, TMPQHash * pHash, TMPQBlock * pBlockTable)
 {
     ULONGLONG ByteOffset;    
@@ -592,13 +604,6 @@ static bool IsValidHashEntry1(TMPQArchive * ha, TMPQHash * pHash, TMPQBlock * pB
     return false;
 }
 
-// Hash entry verification when the file table does not exist yet
-static bool IsValidHashEntry2(TMPQArchive * ha, TMPQHash * pHash)
-{
-    TFileEntry * pFileEntry = ha->pFileTable + pHash->dwBlockIndex;
-    return ((pHash->dwBlockIndex < ha->dwFileTableSize) && (pFileEntry->dwFlags & MPQ_FILE_EXISTS)) ? true : false;
-}
-
 // Returns a hash table entry in the following order:
 // 1) A hash table entry with the preferred locale
 // 2) A hash table entry with the neutral locale
@@ -704,6 +709,7 @@ static TMPQHash * DefragmentHashTable(
     if(dwNewTableSize < pHeader->dwHashTableSize)
     {
         pHashTable = STORM_REALLOC(TMPQHash, pHashTable, dwNewTableSize);
+        ha->pHeader->BlockTableSize64 = dwNewTableSize * sizeof(TMPQHash);
         ha->pHeader->dwHashTableSize = dwNewTableSize;
     }
 
@@ -814,10 +820,13 @@ static int BuildFileTableFromBlockTable(
         if(ha->dwFileTableSize > ha->dwMaxFileCount)
         {
             ha->pFileTable = STORM_REALLOC(TFileEntry, ha->pFileTable, ha->dwMaxFileCount);
+            ha->pHeader->BlockTableSize64 = ha->dwMaxFileCount * sizeof(TMPQBlock);
             ha->pHeader->dwBlockTableSize = ha->dwMaxFileCount;
             ha->dwFileTableSize = ha->dwMaxFileCount;
         }
 
+//      DumpFileTable(ha->pFileTable, ha->dwFileTableSize);
+
         // Free the translation table
         STORM_FREE(DefragmentTable);
     }
@@ -2697,7 +2706,7 @@ int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize)
         // Parse the old hash table and copy all entries to the new table
         for(pHash = pOldHashTable; pHash < pHashTableEnd; pHash++)
         {
-            if(IsValidHashEntry2(ha, pHash))
+            if(IsValidHashEntry(ha, pHash))
             {
                 pFileEntry = ha->pFileTable + pHash->dwBlockIndex;
                 AllocateHashEntry(ha, pFileEntry, pHash->lcLocale);
diff --git a/src/SFileAddFile.cpp b/src/SFileAddFile.cpp
index 70a857b..ea3c5f9 100644
--- a/src/SFileAddFile.cpp
+++ b/src/SFileAddFile.cpp
@@ -80,6 +80,45 @@ static bool IsWaveFile_16BitsPerAdpcmSample(
     return false;
 }
 
+static int FillWritableHandle(
+    TMPQArchive * ha,
+    TMPQFile * hf,
+    ULONGLONG FileTime,
+    DWORD dwFileSize,
+    DWORD dwFlags)
+{
+    TFileEntry * pFileEntry = hf->pFileEntry;
+
+    // Initialize the hash entry for the file
+    hf->RawFilePos = ha->MpqPos + hf->MpqFilePos;
+    hf->dwDataSize = dwFileSize;
+
+    // Initialize the block table entry for the file
+    pFileEntry->ByteOffset = hf->MpqFilePos;
+    pFileEntry->dwFileSize = dwFileSize;
+    pFileEntry->dwCmpSize = 0;
+    pFileEntry->dwFlags  = dwFlags | MPQ_FILE_EXISTS;
+
+    // Initialize the file time, CRC32 and MD5
+    assert(sizeof(hf->hctx) >= sizeof(hash_state));
+    memset(pFileEntry->md5, 0, MD5_DIGEST_SIZE);
+    md5_init((hash_state *)hf->hctx);
+    pFileEntry->dwCrc32 = crc32(0, Z_NULL, 0);
+
+    // If the caller gave us a file time, use it.
+    pFileEntry->FileTime = FileTime;
+
+    // Mark the archive as modified
+    ha->dwFlags |= MPQ_FLAG_CHANGED;
+
+    // Call the callback, if needed
+    if(ha->pfnAddFileCB != NULL)
+        ha->pfnAddFileCB(ha->pvAddFileUserData, 0, hf->dwDataSize, false);
+    hf->nAddFileError = ERROR_SUCCESS;
+
+    return ERROR_SUCCESS;
+}
+
 //-----------------------------------------------------------------------------
 // MPQ write data functions
 
@@ -366,7 +405,6 @@ int SFileAddFile_Init(
     TMPQFile ** phf)
 {
     TFileEntry * pFileEntry = NULL;
-    ULONGLONG TempPos;                  // For various file offset calculations
     TMPQFile * hf = NULL;               // File structure for newly added file
     DWORD dwHashIndex = HASH_ENTRY_FREE;
     int nError = ERROR_SUCCESS;
@@ -395,28 +433,9 @@ int SFileAddFile_Init(
         lcLocale = 0;
 
     // Allocate the TMPQFile entry for newly added file
-    hf = CreateFileHandle(ha, NULL);
+    hf = CreateWritableHandle(ha, dwFileSize);
     if(hf == NULL)
-        nError = ERROR_NOT_ENOUGH_MEMORY;
-
-    // Find a free space in the MPQ and verify if it's not over 4 GB on MPQs v1
-    if(nError == ERROR_SUCCESS)
-    {
-        // Find the position where the file will be stored
-        hf->MpqFilePos = FindFreeMpqSpace(ha);
-        hf->RawFilePos = ha->MpqPos + hf->MpqFilePos;
-        hf->bIsWriteHandle = true;
-
-        // When format V1, the size of the archive cannot exceed 4 GB
-        if(ha->pHeader->wFormatVersion == MPQ_FORMAT_VERSION_1)
-        {
-            TempPos  = hf->MpqFilePos + dwFileSize;
-            TempPos += ha->pHeader->dwHashTableSize * sizeof(TMPQHash);
-            TempPos += ha->dwFileTableSize * sizeof(TMPQBlock);
-            if((TempPos >> 32) != 0)
-                nError = ERROR_DISK_FULL;
-        }
-    }
+        nError = GetLastError();
 
     // Allocate file entry in the MPQ
     if(nError == ERROR_SUCCESS)
@@ -440,6 +459,24 @@ int SFileAddFile_Init(
             if(pFileEntry == NULL)
                 nError = ERROR_DISK_FULL;   
         }
+
+        // Set the file entry to the file structure
+        hf->pFileEntry = pFileEntry;
+    }
+
+    // Prepare the pointer to hash table entry
+    if(nError == ERROR_SUCCESS && ha->pHashTable != NULL && dwHashIndex < ha->pHeader->dwHashTableSize)
+    {
+        hf->pHashEntry = ha->pHashTable + dwHashIndex;
+        hf->pHashEntry->lcLocale = (USHORT)lcLocale;
+    }
+
+    // Prepare the file key
+    if(nError == ERROR_SUCCESS && (dwFlags & MPQ_FILE_ENCRYPTED))
+    {
+        hf->dwFileKey = DecryptFileKey(szFileName, hf->MpqFilePos, dwFileSize, dwFlags);
+        if(hf->dwFileKey == 0)
+            nError = ERROR_UNKNOWN_FILE_KEY;
     }
 
     // Fill the file entry and TMPQFile structure
@@ -449,46 +486,77 @@ int SFileAddFile_Init(
         assert(pFileEntry->szFileName != NULL);
         assert(_stricmp(pFileEntry->szFileName, szFileName) == 0);
 
-        // Initialize the hash entry for the file
-        hf->pFileEntry = pFileEntry;
-        hf->dwDataSize = dwFileSize;
+        nError = FillWritableHandle(ha, hf, FileTime, dwFileSize, dwFlags);
+    }
 
-        // Set the hash table entry
-        if(ha->pHashTable != NULL && dwHashIndex < ha->pHeader->dwHashTableSize)
-        {
-            hf->pHashEntry = ha->pHashTable + dwHashIndex;
-            hf->pHashEntry->lcLocale = (USHORT)lcLocale;
-        }
+    // Free the file handle if failed
+    if(nError != ERROR_SUCCESS && hf != NULL)
+        FreeFileHandle(hf);
+
+    // Give the handle to the caller
+    *phf = hf;
+    return nError;
+}
+
+int SFileAddFile_Init(
+    TMPQArchive * ha,
+    TMPQFile * hfSrc,
+    TMPQFile ** phf)
+{
+    TFileEntry * pFileEntry = NULL;
+    TMPQFile * hf = NULL;               // File structure for newly added file
+    ULONGLONG FileTime = hfSrc->pFileEntry->FileTime;
+    DWORD dwFileSize = hfSrc->pFileEntry->dwFileSize;
+    DWORD dwFlags = hfSrc->pFileEntry->dwFlags;
+    int nError = ERROR_SUCCESS;
+
+    // Allocate the TMPQFile entry for newly added file
+    hf = CreateWritableHandle(ha, dwFileSize);
+    if(hf == NULL)
+        nError = ERROR_NOT_ENOUGH_MEMORY;
 
-        // Decrypt the file key
-        if(dwFlags & MPQ_FILE_ENCRYPTED)
-            hf->dwFileKey = DecryptFileKey(szFileName, hf->MpqFilePos, dwFileSize, dwFlags);
+    // We need to keep the file entry index the same like in the source archive
+    // This is because multiple hash table entries can point to the same file entry
+    if(nError == ERROR_SUCCESS)
+    {
+        // Retrieve the file entry for the target file
+        pFileEntry = ha->pFileTable + (hfSrc->pFileEntry - hfSrc->ha->pFileTable);
 
-        // Initialize the block table entry for the file
-        pFileEntry->ByteOffset = hf->MpqFilePos;
-        pFileEntry->dwFileSize = dwFileSize;
-        pFileEntry->dwCmpSize = 0;
-        pFileEntry->dwFlags  = dwFlags | MPQ_FILE_EXISTS;
+        // Copy all variables except file name
+        if((pFileEntry->dwFlags & MPQ_FILE_EXISTS) == 0)
+        {
+            pFileEntry[0] = hfSrc->pFileEntry[0];
+            pFileEntry->szFileName = NULL;
+        }
+        else
+            nError = ERROR_ALREADY_EXISTS;
 
-        // Initialize the file time, CRC32 and MD5
-        assert(sizeof(hf->hctx) >= sizeof(hash_state));
-        memset(pFileEntry->md5, 0, MD5_DIGEST_SIZE);
-        md5_init((hash_state *)hf->hctx);
-        pFileEntry->dwCrc32 = crc32(0, Z_NULL, 0);
+        // Set the file entry to the file structure
+        hf->pFileEntry = pFileEntry;
+    }
 
-        // If the caller gave us a file time, use it.
-        pFileEntry->FileTime = FileTime;
+    // Prepare the pointer to hash table entry
+    if(nError == ERROR_SUCCESS && ha->pHashTable != NULL && hfSrc->pHashEntry != NULL)
+    {
+        hf->dwHashIndex = (DWORD)(hfSrc->pHashEntry - hfSrc->ha->pHashTable);
+        hf->pHashEntry = ha->pHashTable + hf->dwHashIndex;
+    }
 
-        // Mark the archive as modified
-        ha->dwFlags |= MPQ_FLAG_CHANGED;
+    // Prepare the file key (copy from source file)
+    if(nError == ERROR_SUCCESS && (dwFlags & MPQ_FILE_ENCRYPTED))
+    {
+        hf->dwFileKey = hfSrc->dwFileKey;
+        if(hf->dwFileKey == 0)
+            nError = ERROR_UNKNOWN_FILE_KEY;
+    }
 
-        // Call the callback, if needed
-        if(ha->pfnAddFileCB != NULL)
-            ha->pfnAddFileCB(ha->pvAddFileUserData, 0, hf->dwDataSize, false);
-        hf->nAddFileError = ERROR_SUCCESS;
+    // Fill the file entry and TMPQFile structure
+    if(nError == ERROR_SUCCESS)
+    {
+        nError = FillWritableHandle(ha, hf, FileTime, dwFileSize, dwFlags);
     }
 
-    // Fre the file handle if failed
+    // Free the file handle if failed
     if(nError != ERROR_SUCCESS && hf != NULL)
         FreeFileHandle(hf);
 
diff --git a/src/SFileAttributes.cpp b/src/SFileAttributes.cpp
index bd1956d..d37d83c 100644
--- a/src/SFileAttributes.cpp
+++ b/src/SFileAttributes.cpp
@@ -424,9 +424,6 @@ int SAttrFileSaveToMpq(TMPQArchive * ha)
         pbAttrFile = CreateAttributesFile(ha, &cbAttrFile);
         if(pbAttrFile != NULL)
         {
-            // We expect it to be nonzero size
-            assert(cbAttrFile != 0);
-
             // Determine the real flags for (attributes)
             if(ha->dwFileFlags2 == MPQ_FILE_EXISTS)
                 ha->dwFileFlags2 = GetDefaultSpecialFileFlags(cbAttrFile, ha->pHeader->wFormatVersion);
diff --git a/src/SFileCompactArchive.cpp b/src/SFileCompactArchive.cpp
index 2c0ef5a..be1ca66 100644
--- a/src/SFileCompactArchive.cpp
+++ b/src/SFileCompactArchive.cpp
@@ -81,32 +81,7 @@ static int CheckIfAllKeysKnown(TMPQArchive * ha, const char * szListFile, LPDWOR
                                                              pFileEntry->dwFlags);
                     continue;
                 }
-/*
-                // If the file has a nonzero size, we can try to read few bytes of data
-                // and force to detect the decryption key that way
-                if(pFileEntry->dwFileSize > 0x10)
-                {
-                    TMPQFile * hf = NULL;
-                    DWORD dwBytesRead = 0;
-                    DWORD FileData[4];
-
-                    // Create file handle where we load the sector offset table
-                    hf = CreateFileHandle(ha, pFileEntry);
-                    if(hf != NULL)
-                    {
-                        // Call one dummy load of the first 4 bytes.
-                        // This enforces loading all buffers and also detecting of the decryption key
-                        SFileReadFile((HANDLE)hf, FileData, sizeof(FileData), &dwBytesRead, NULL);
-                        pFileKeys[dwBlockIndex] = hf->dwFileKey;
-                        FreeFileHandle(hf);
-                    }
-
-                    // If we succeeded in reading 16 bytes from the file,
-                    // we also know the encryption key
-                    if(dwBytesRead == sizeof(FileData))
-                        continue;
-                }
-*/
+
                 // We don't know the encryption key of this file,
                 // thus we cannot compact the file
                 nError = ERROR_UNKNOWN_FILE_NAMES;
@@ -471,11 +446,68 @@ static int CopyMpqFiles(TMPQArchive * ha, LPDWORD pFileKeys, TFileStream * pNewS
     return nError;
 }
 
-
 /*****************************************************************************/
 /* Public functions                                                          */
 /*****************************************************************************/
 
+//-----------------------------------------------------------------------------
+// Changing hash table size
+
+DWORD WINAPI SFileGetMaxFileCount(HANDLE hMpq)
+{
+    TMPQArchive * ha = (TMPQArchive *)hMpq;
+
+    return ha->dwMaxFileCount;
+}
+
+bool WINAPI SFileSetMaxFileCount(HANDLE hMpq, DWORD dwMaxFileCount)
+{
+    TMPQArchive * ha = (TMPQArchive *)hMpq;
+    DWORD dwNewHashTableSize = 0;
+    int nError = ERROR_SUCCESS;
+
+    // Test the valid parameters
+    if(!IsValidMpqHandle(hMpq))
+        nError = ERROR_INVALID_HANDLE;
+    if(ha->dwFlags & MPQ_FLAG_READ_ONLY)
+        nError = ERROR_ACCESS_DENIED;
+    if(dwMaxFileCount < ha->dwFileTableSize)
+        nError = ERROR_DISK_FULL;
+
+    // ALL file names must be known in order to be able to rebuild hash table
+    if(nError == ERROR_SUCCESS && ha->pHashTable != NULL)
+    {
+        nError = CheckIfAllFilesKnown(ha);
+        if(nError == ERROR_SUCCESS)
+        {
+            // Calculate the hash table size for the new file limit
+            dwNewHashTableSize = GetHashTableSizeForFileCount(dwMaxFileCount);
+
+            // Rebuild both file tables
+            nError = RebuildFileTable(ha, dwNewHashTableSize);
+        }
+    }
+
+    // We always have to rebuild the (attributes) file due to file table change
+    if(nError == ERROR_SUCCESS)
+    {
+        // Invalidate (listfile) and (attributes)
+        InvalidateInternalFiles(ha);
+
+        // Rebuild the HET table, if we have any
+        if(ha->pHetTable != NULL)
+            nError = RebuildHetTable(ha);
+    }
+
+    // Return the error
+    if(nError != ERROR_SUCCESS)
+        SetLastError(nError);
+    return (nError == ERROR_SUCCESS);
+}
+
+//-----------------------------------------------------------------------------
+// Archive compacting
+
 bool WINAPI SFileSetCompactCallback(HANDLE hMpq, SFILE_COMPACT_CALLBACK pfnCompactCB, void * pvUserData)
 {
     TMPQArchive * ha = (TMPQArchive *) hMpq;
@@ -491,9 +523,6 @@ bool WINAPI SFileSetCompactCallback(HANDLE hMpq, SFILE_COMPACT_CALLBACK pfnCompa
     return true;
 }
 
-//-----------------------------------------------------------------------------
-// Archive compacting
-
 bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bReserved */)
 {
     TFileStream * pTempStream = NULL;
@@ -625,58 +654,3 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR
         SetLastError(nError);
     return (nError == ERROR_SUCCESS);
 }
-
-//-----------------------------------------------------------------------------
-// Changing hash table size
-
-DWORD WINAPI SFileGetMaxFileCount(HANDLE hMpq)
-{
-    TMPQArchive * ha = (TMPQArchive *)hMpq;
-
-    return ha->dwMaxFileCount;
-}
-
-bool WINAPI SFileSetMaxFileCount(HANDLE hMpq, DWORD dwMaxFileCount)
-{
-    TMPQArchive * ha = (TMPQArchive *)hMpq;
-    DWORD dwNewHashTableSize = 0;
-    int nError = ERROR_SUCCESS;
-
-    // Test the valid parameters
-    if(!IsValidMpqHandle(hMpq))
-        nError = ERROR_INVALID_HANDLE;
-    if(ha->dwFlags & MPQ_FLAG_READ_ONLY)
-        nError = ERROR_ACCESS_DENIED;
-    if(dwMaxFileCount < ha->dwFileTableSize)
-        nError = ERROR_DISK_FULL;
-
-    // ALL file names must be known in order to be able to rebuild hash table
-    if(nError == ERROR_SUCCESS && ha->pHashTable != NULL)
-    {
-        nError = CheckIfAllFilesKnown(ha);
-        if(nError == ERROR_SUCCESS)
-        {
-            // Calculate the hash table size for the new file limit
-            dwNewHashTableSize = GetHashTableSizeForFileCount(dwMaxFileCount);
-
-            // Rebuild both file tables
-            nError = RebuildFileTable(ha, dwNewHashTableSize);
-        }
-    }
-
-    // We always have to rebuild the (attributes) file due to file table change
-    if(nError == ERROR_SUCCESS)
-    {
-        // Invalidate (listfile) and (attributes)
-        InvalidateInternalFiles(ha);
-
-        // Rebuild the HET table, if we have any
-        if(ha->pHetTable != NULL)
-            nError = RebuildHetTable(ha);
-    }
-
-    // Return the error
-    if(nError != ERROR_SUCCESS)
-        SetLastError(nError);
-    return (nError == ERROR_SUCCESS);
-}
diff --git a/src/SFileListFile.cpp b/src/SFileListFile.cpp
index 90f71fa..f0d2122 100644
--- a/src/SFileListFile.cpp
+++ b/src/SFileListFile.cpp
@@ -77,22 +77,22 @@ static TListFileCache * CreateListFileCache(HANDLE hListFile, const char * szWil
         memset(pCache, 0, sizeof(TListFileCache) + cchWildCard);
 
         // Shall we copy the mask?
-        if(cchWildCard != NULL)
+        if(cchWildCard != 0)
         {
             pCache->szWildCard = (char *)(pCache + 1);
             memcpy(pCache->szWildCard, szWildCard, cchWildCard);
         }
                           
         // Fill-in the rest of the cache pointers
-        pCache->pBegin = (LPBYTE)(pCache + 1) + cchWildCard + 1;
+        pCache->pBegin = (LPBYTE)(pCache + 1) + cchWildCard;
 
         // Load the entire listfile to the cache
         SFileReadFile(hListFile, pCache->pBegin, dwFileSize, &dwBytesRead, NULL);
-        if(dwFileSize != 0)
+        if(dwBytesRead != 0)
         {
             // Allocate pointers
             pCache->pPos = pCache->pBegin;
-            pCache->pEnd = pCache->pBegin + dwFileSize;
+            pCache->pEnd = pCache->pBegin + dwBytesRead;
         }
         else
         {
@@ -302,6 +302,11 @@ static LPBYTE CreateListFile(TMPQArchive * ha, DWORD * pcbListFile)
             assert((size_t)(szListLine - szListFile) == cbListFile);
         }
     }
+    else
+    {
+        szListFile = STORM_ALLOC(char, 1);
+        cbListFile = 0;
+    }
 
     // Free the sort table
     STORM_FREE(SortTable);
@@ -379,9 +384,6 @@ int SListFileSaveToMpq(TMPQArchive * ha)
         pbListFile = CreateListFile(ha, &cbListFile);
         if(pbListFile != NULL)
         {
-            // We expect it to be nonzero size
-            assert(cbListFile != 0);
-
             // Determine the real flags for (listfile)
             if(ha->dwFileFlags1 == MPQ_FILE_EXISTS)
                 ha->dwFileFlags1 = GetDefaultSpecialFileFlags(cbListFile, ha->pHeader->wFormatVersion);
diff --git a/src/SFileReadFile.cpp b/src/SFileReadFile.cpp
index 3293cf6..d81b4de 100644
--- a/src/SFileReadFile.cpp
+++ b/src/SFileReadFile.cpp
@@ -166,6 +166,10 @@ static int ReadMpqSectors(TMPQFile * hf, LPBYTE pbBuffer, DWORD dwByteOffset, DW
                 // Is the file compressed by Blizzard's multiple compression ?
                 if(pFileEntry->dwFlags & MPQ_FILE_COMPRESS)
                 {
+                    // Remember the last used compression
+                    hf->dwCompression0 = pbInSector[0];
+
+                    // Decompress the data
                     if(ha->pHeader->wFormatVersion >= MPQ_FORMAT_VERSION_2)
                         nResult = SCompDecompress2(pbOutSector, &cbOutSector, pbInSector, cbInSector);
                     else
@@ -289,6 +293,10 @@ static int ReadMpqFileSingleUnit(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos
             // Is the file compressed by Blizzard's multiple compression ?
             if(pFileEntry->dwFlags & MPQ_FILE_COMPRESS)
             {
+                // Remember the last used compression
+                hf->dwCompression0 = pbRawData[0];
+
+                // Decompress the file
                 if(ha->pHeader->wFormatVersion >= MPQ_FORMAT_VERSION_2)
                     nResult = SCompDecompress2(hf->pbFileSector, &cbOutBuffer, pbRawData, cbInBuffer);
                 else
@@ -395,6 +403,7 @@ static int ReadMpkFileSingleUnit(TMPQFile * hf, void * pvBuffer, DWORD dwFilePos
         {
             int cbOutBuffer = (int)hf->dwDataSize;
 
+            hf->dwCompression0 = pbRawData[0];
             if(!SCompDecompressMpk(hf->pbFileSector, &cbOutBuffer, pbRawData, (int)pFileEntry->dwCmpSize))
                 nError = ERROR_FILE_CORRUPT;
         }
@@ -688,6 +697,9 @@ bool WINAPI SFileReadFile(HANDLE hFile, void * pvBuffer, DWORD dwToRead, LPDWORD
         }
     }
 
+    // Clear the last used compression
+    hf->dwCompression0 = 0;
+
     // If the file is local file, read the data directly from the stream
     if(hf->pStream != NULL)
     {
diff --git a/src/StormCommon.h b/src/StormCommon.h
index 3363a70..d4bf57c 100644
--- a/src/StormCommon.h
+++ b/src/StormCommon.h
@@ -178,6 +178,8 @@ ULONGLONG CalculateRawSectorOffset(TMPQFile * hf, DWORD dwSectorOffset);
 
 int ConvertMpqHeaderToFormat4(TMPQArchive * ha, ULONGLONG MpqOffset, ULONGLONG FileSize, DWORD dwFlags);
 
+bool IsValidHashEntry(TMPQArchive * ha, TMPQHash * pHash);
+
 TMPQHash * FindFreeHashEntry(TMPQArchive * ha, DWORD dwStartIndex, DWORD dwName1, DWORD dwName2, LCID lcLocale);
 TMPQHash * GetFirstHashEntry(TMPQArchive * ha, const char * szFileName);
 TMPQHash * GetNextHashEntry(TMPQArchive * ha, TMPQHash * pFirstHash, TMPQHash * pPrevHash);
@@ -245,6 +247,7 @@ int SCompDecompressMpk(void * pvOutBuffer, int * pcbOutBuffer, void * pvInBuffer
 // Common functions - MPQ File
 
 TMPQFile * CreateFileHandle(TMPQArchive * ha, TFileEntry * pFileEntry);
+TMPQFile * CreateWritableHandle(TMPQArchive * ha, DWORD dwFileSize);
 void * LoadMpqTable(TMPQArchive * ha, ULONGLONG ByteOffset, DWORD dwCompressedSize, DWORD dwRealSize, DWORD dwKey, bool * pbTableIsCut);
 int  AllocateSectorBuffer(TMPQFile * hf);
 int  AllocatePatchInfo(TMPQFile * hf, bool bLoadFromFile);
@@ -303,6 +306,12 @@ int SFileAddFile_Init(
     TMPQFile ** phf
     );
 
+int SFileAddFile_Init(
+    TMPQArchive * ha,
+    TMPQFile * hfSrc,
+    TMPQFile ** phf
+    );
+
 int SFileAddFile_Write(
     TMPQFile * hf,
     const void * pvData,
@@ -339,12 +348,14 @@ int SSignFileFinish(TMPQArchive * ha);
 void DumpMpqHeader(TMPQHeader * pHeader);
 void DumpHashTable(TMPQHash * pHashTable, DWORD dwHashTableSize);
 void DumpHetAndBetTable(TMPQHetTable * pHetTable, TMPQBetTable * pBetTable);
+void DumpFileTable(TFileEntry * pFileTable, DWORD dwFileTableSize);
 
 #else
 
-#define DumpMpqHeader(h)           /* */
-#define DumpHashTable(h, s)        /* */
-#define DumpHetAndBetTable(h, b)   /* */
+#define DumpMpqHeader(h)            /* */
+#define DumpHashTable(t, s)         /* */
+#define DumpHetAndBetTable(t, s)    /* */
+#define DumpFileTable(t, s)         /* */
 
 #endif
 
diff --git a/test/StormTest.cpp b/test/StormTest.cpp
index 06e599f..f9d707a 100644
--- a/test/StormTest.cpp
+++ b/test/StormTest.cpp
@@ -288,6 +288,30 @@ static bool IsMpqExtension(const char * szFileName)
     return false;
 }
 
+static void AddStringBeforeExtension(char * szBuffer, const char * szFileName, const char * szExtraString)
+{
+    const char * szExtension;
+    size_t nLength;
+
+    // Get the extension
+    szExtension = strrchr(szFileName, '.');
+    if(szExtension == NULL)
+        szExtension = szFileName + strlen(szFileName);
+    nLength = (size_t)(szExtension - szFileName);
+
+    // Copy the part before extension
+    memcpy(szBuffer, szFileName, nLength);
+    szFileName += nLength;
+    szBuffer += nLength;
+
+    // Append the extra data
+    if(szExtraString != NULL)
+        strcpy(szBuffer, szExtraString);
+
+    // Append the rest of the file name
+    strcat(szBuffer, szFileName);
+}
+
 static bool CompareBlocks(LPBYTE pbBlock1, LPBYTE pbBlock2, DWORD dwLength, DWORD * pdwDifference)
 {
     for(DWORD i = 0; i < dwLength; i++)
@@ -2357,7 +2381,7 @@ static int TestOpenArchive(const char * szPlainName, const char * szListFile = N
     bool bIsPartialMpq = false;
     int nError;
 
-    // If the file is a partial MPQ, don;t load all files
+    // If the file is a partial MPQ, don't load all files
     bIsPartialMpq = (strstr(szPlainName, ".MPQ.part") != NULL);
 
     // Copy the archive so we won't fuck up the original one
@@ -3069,7 +3093,77 @@ static int TestAddFile_ListFileTest(const char * szSourceMpq, bool bShouldHaveLi
         SFileCloseArchive(hMpq);
     return nError;
 }
+/*
+static int TestCreateArchive_Deprotect(const char * szPlainName)
+{
+    TLogHelper Logger("DeprotectTest", szPlainName);
+    HANDLE hMpq1 = NULL;
+    HANDLE hMpq2 = NULL;
+    char szMpqName1[MAX_PATH];
+    char szMpqName2[MAX_PATH];
+    BYTE FileHash1[MD5_DIGEST_SIZE];
+    BYTE FileHash2[MD5_DIGEST_SIZE];
+    DWORD dwFileCount1 = 0;
+    DWORD dwFileCount2 = 0;
+    DWORD dwTestFlags = TEST_FLAG_LOAD_FILES | TEST_FLAG_HASH_FILES;
+    int nError = ERROR_SUCCESS;
+
+    // First copy: The original (untouched) file
+    if(nError == ERROR_SUCCESS)
+    {
+        AddStringBeforeExtension(szMpqName1, szPlainName, "_original");
+        nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szMpqName1, &hMpq1);
+        if(nError != ERROR_SUCCESS)
+            Logger.PrintMessage("Failed to open %s", szMpqName1);
+    }
+
+    // Second copy: Will be deprotected
+    if(nError == ERROR_SUCCESS)
+    {
+        AddStringBeforeExtension(szMpqName2, szPlainName, "_deprotected");
+        nError = OpenExistingArchiveWithCopy(&Logger, szPlainName, szMpqName2, &hMpq2);
+        if(nError != ERROR_SUCCESS)
+            Logger.PrintMessage("Failed to open %s", szMpqName2);
+    }
 
+    // Deprotect the second map
+    if(nError == ERROR_SUCCESS)
+    {
+        SFileSetCompactCallback(hMpq2, CompactCallback, &Logger);
+        if(!SFileCompactArchive(hMpq2, NULL, false))
+            nError = Logger.PrintError("Failed to deprotect archive %s", szMpqName2);
+    }
+
+    // Calculate number of files and compare their hash (archive 1)
+    if(nError == ERROR_SUCCESS)
+    {
+        memset(FileHash1, 0, sizeof(FileHash1));
+        nError = SearchArchive(&Logger, hMpq1, dwTestFlags, &dwFileCount1, FileHash1);
+    }
+
+    // Calculate number of files and compare their hash (archive 2)
+    if(nError == ERROR_SUCCESS)
+    {
+        memset(FileHash1, 0, sizeof(FileHash2));
+        nError = SearchArchive(&Logger, hMpq2, dwTestFlags, &dwFileCount2, FileHash2);
+    }
+
+    if(nError == ERROR_SUCCESS)
+    {
+        if(dwFileCount1 != dwFileCount2)
+            Logger.PrintMessage("Different file count (%u in %s; %u in %s)", dwFileCount1, szMpqName1, dwFileCount2, szMpqName2); 
+        if(memcmp(FileHash1, FileHash2, MD5_DIGEST_SIZE))
+            Logger.PrintMessage("Different file hash (%s vs %s)", szMpqName1, szMpqName2);
+    }
+
+    // Close both MPQs
+    if(hMpq2 != NULL)
+        SFileCloseArchive(hMpq2);
+    if(hMpq1 != NULL)
+        SFileCloseArchive(hMpq1);
+    return nError;
+}
+*/
 static int TestCreateArchive_EmptyMpq(const char * szPlainName, DWORD dwCreateFlags)
 {
     TLogHelper Logger("CreateEmptyMpq", szPlainName);
@@ -4029,7 +4123,7 @@ int main(int argc, char * argv[])
     // Search in listfile
     if(nError == ERROR_SUCCESS)
         nError = TestSearchListFile("ListFile_Blizzard.txt");
-/*
+
     // Test opening local file with SFileOpenFileEx
     if(nError == ERROR_SUCCESS)
         nError = TestOpenLocalFile("ListFile_Blizzard.txt");
@@ -4057,23 +4151,44 @@ int main(int argc, char * argv[])
     // Open a truncated archive
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2002_v1_BlockTableCut.MPQ");
-*/
-    // Open an Warcraft III map locked by a protector
+
+    // Open a MPQ that actually has user data
+    if(nError == ERROR_SUCCESS)
+        nError = TestOpenArchive("MPQ_2010_v2_HasUserData.s2ma");
+
+    // Open an Warcraft III map whose "(attributes)" file has (BlockTableSize-1) entries
+    if(nError == ERROR_SUCCESS)
+        nError = TestOpenArchive("MPQ_2014_v1_AttributesOneEntryLess.w3x");
+
+    // Open a MPQ archive v 3.0
+    if(nError == ERROR_SUCCESS)
+        nError = TestOpenArchive("MPQ_2010_v3_expansion-locale-frFR.MPQ");
+
+    // Open an encrypted archive from Starcraft II installer
+    if(nError == ERROR_SUCCESS)
+        nError = TestOpenArchive("mpqe-file://MPQ_2011_v2_EncryptedMpq.MPQE");
+
+    // Open a MPK archive from Longwu online
+    if(nError == ERROR_SUCCESS)
+        nError = TestOpenArchive("MPx_2013_v1_LongwuOnline.mpk");
+
+    // Open a SQP archive from War of the Immortals
+    if(nError == ERROR_SUCCESS)
+        nError = TestOpenArchive("MPx_2013_v1_WarOfTheImmortals.sqp", "ListFile_WarOfTheImmortals.txt");
+
+    // Open a partial MPQ with compressed hash table
+    if(nError == ERROR_SUCCESS)
+        nError = TestOpenArchive("part-file://MPQ_2010_v2_HashTableCompressed.MPQ.part");
+
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_HashTable_FakeValid.w3x");
-/*
-    // Open an Warcraft III map locked by a protector
+
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_InvalidUserData.w3x");
 
-    // Open an Warcraft III map locked by a protector
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_InvalidMpqFormat.w3x");
 
-    // Open a MPQ that actually has user data
-    if(nError == ERROR_SUCCESS)
-        nError = TestOpenArchive("MPQ_2010_v2_HasUserData.s2ma");
-
     // Open an Warcraft III map locked by the Spazzler protector
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2002_v1_ProtectedMap_Spazzler.w3x");
@@ -4104,30 +4219,6 @@ int main(int argc, char * argv[])
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive("MPQ_2015_v1_flem1.w3x");
 
-    // Open an Warcraft III map whose "(attributes)" file has (BlockTableSize-1) entries
-    if(nError == ERROR_SUCCESS)
-        nError = TestOpenArchive("MPQ_2014_v1_AttributesOneEntryLess.w3x");
-
-    // Open a MPQ archive v 3.0
-    if(nError == ERROR_SUCCESS)
-        nError = TestOpenArchive("MPQ_2010_v3_expansion-locale-frFR.MPQ");
-
-    // Open an encrypted archive from Starcraft II installer
-    if(nError == ERROR_SUCCESS)
-        nError = TestOpenArchive("mpqe-file://MPQ_2011_v2_EncryptedMpq.MPQE");
-
-    // Open a MPK archive from Longwu online
-    if(nError == ERROR_SUCCESS)
-        nError = TestOpenArchive("MPx_2013_v1_LongwuOnline.mpk");
-
-    // Open a SQP archive from War of the Immortals
-    if(nError == ERROR_SUCCESS)
-        nError = TestOpenArchive("MPx_2013_v1_WarOfTheImmortals.sqp", "ListFile_WarOfTheImmortals.txt");
-
-    // Open a partial MPQ with compressed hash table
-    if(nError == ERROR_SUCCESS)
-        nError = TestOpenArchive("part-file://MPQ_2010_v2_HashTableCompressed.MPQ.part");
-
     // Open the multi-file archive with wrong prefix to see how StormLib deals with it
     if(nError == ERROR_SUCCESS)
         nError = TestOpenArchive_WillFail("flat-file://streaming/model.MPQ.0");
@@ -4312,7 +4403,7 @@ int main(int argc, char * argv[])
     // Test replacing a file with zero size file
     if(nError == ERROR_SUCCESS)
         nError = TestModifyArchive_ReplaceFile("MPQ_2014_v4_Base.StormReplay", "AddFile-replay.message.events");
-*/
+
 #ifdef _MSC_VER
     _CrtDumpMemoryLeaks();
 #endif  // _MSC_VER