+ Optimized loading listfile

3 files changed, 99 insertions, 194 deletions

diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp
index d7e0702..3e98ebb 100644
--- a/src/SBaseFileTable.cpp
+++ b/src/SBaseFileTable.cpp
@@ -787,7 +787,7 @@ static int BuildFileTableFromBlockTable(
                 pHash->dwBlockIndex = dwNewIndex;
 
                 // Dump the relocation entry
-                printf("Relocating hash entry %08X-%08X: %08X -> %08X\n", pHash->dwName1, pHash->dwName2, dwBlockIndex, dwNewIndex);
+//              printf("Relocating hash entry %08X-%08X: %08X -> %08X\n", pHash->dwName1, pHash->dwName2, dwBlockIndex, dwNewIndex);
             }
 
             // Get the pointer to the file entry and the block entry
diff --git a/src/SFileListFile.cpp b/src/SFileListFile.cpp
index 385098e..90f71fa 100644
--- a/src/SFileListFile.cpp
+++ b/src/SFileListFile.cpp
@@ -21,16 +21,13 @@
 
 struct TListFileCache
 {
-    HANDLE  hFile;                      // Stormlib file handle
-    char  * szMask;                     // Self-relative pointer to file mask
-    DWORD   dwFileSize;                 // Total size of the cached file
-    DWORD   dwFilePos;                  // Position of the cache in the file
-    BYTE  * pBegin;                     // The begin of the listfile cache
-    BYTE  * pPos;
-    BYTE  * pEnd;                       // The last character in the file cache
-
-    BYTE Buffer[CACHE_BUFFER_SIZE];
-//  char MaskBuff[1]                    // Followed by the name mask (if any)
+    char * szWildCard;                  // Self-relative pointer to file mask
+    LPBYTE pBegin;                      // The begin of the listfile cache
+    LPBYTE pPos;                        // Current position in the cache
+    LPBYTE pEnd;                        // The last character in the file cache
+
+//  char szWildCard[wildcard_length];   // Followed by the name mask (if any)
+//  char szListFile[listfile_length];   // Followed by the listfile (if any)
 };
 
 //-----------------------------------------------------------------------------
@@ -56,45 +53,46 @@ static bool FreeListFileCache(TListFileCache * pCache)
     return true;
 }
 
-static TListFileCache * CreateListFileCache(HANDLE hListFile, const char * szMask)
+static TListFileCache * CreateListFileCache(HANDLE hListFile, const char * szWildCard)
 {
     TListFileCache * pCache = NULL;
-    size_t nMaskLength = 0;
+    size_t cchWildCard = 0;
     DWORD dwBytesRead = 0;
     DWORD dwFileSize;
 
     // Get the amount of bytes that need to be allocated
     dwFileSize = SFileGetFileSize(hListFile, NULL);
-    if(dwFileSize == 0)
+    if(dwFileSize == 0 || dwFileSize > MAX_LISTFILE_SIZE)
         return NULL;
 
     // Append buffer for name mask, if any
-    if(szMask != NULL)
-        nMaskLength = strlen(szMask) + 1;
+    if(szWildCard != NULL)
+        cchWildCard = strlen(szWildCard) + 1;
 
     // Allocate cache for one file block
-    pCache = (TListFileCache *)STORM_ALLOC(BYTE, sizeof(TListFileCache) + nMaskLength);
+    pCache = (TListFileCache *)STORM_ALLOC(BYTE, sizeof(TListFileCache) + cchWildCard + dwFileSize + 1);
     if(pCache != NULL)
     {
         // Clear the entire structure
-        memset(pCache, 0, sizeof(TListFileCache) + nMaskLength);
+        memset(pCache, 0, sizeof(TListFileCache) + cchWildCard);
 
         // Shall we copy the mask?
-        if(szMask != NULL)
+        if(cchWildCard != NULL)
         {
-            pCache->szMask = (char *)(pCache + 1);
-            memcpy(pCache->szMask, szMask, nMaskLength);
+            pCache->szWildCard = (char *)(pCache + 1);
+            memcpy(pCache->szWildCard, szWildCard, cchWildCard);
         }
+                          
+        // Fill-in the rest of the cache pointers
+        pCache->pBegin = (LPBYTE)(pCache + 1) + cchWildCard + 1;
 
-        // Load the file cache from the file
-        SFileReadFile(hListFile, pCache->Buffer, CACHE_BUFFER_SIZE, &dwBytesRead, NULL);
-        if(dwBytesRead != 0)
+        // Load the entire listfile to the cache
+        SFileReadFile(hListFile, pCache->pBegin, dwFileSize, &dwBytesRead, NULL);
+        if(dwFileSize != 0)
         {
             // Allocate pointers
-            pCache->pBegin = pCache->pPos = &pCache->Buffer[0];
-            pCache->pEnd   = pCache->pBegin + dwBytesRead;
-            pCache->dwFileSize = dwFileSize;
-            pCache->hFile  = hListFile;
+            pCache->pPos = pCache->pBegin;
+            pCache->pEnd = pCache->pBegin + dwFileSize;
         }
         else
         {
@@ -171,112 +169,52 @@ static void FreeNameCache(TMPQNameCache * pNameCache)
 */
 #endif  // _DEBUG
 
-// Reloads the cache. Returns number of characters
-// that has been loaded into the cache.
-static DWORD ReloadListFileCache(TListFileCache * pCache)
-{
-    DWORD dwBytesToRead;
-    DWORD dwBytesRead = 0;
-
-    // Only do something if the cache is empty
-    if(pCache->pPos >= pCache->pEnd)
-    {
-        // Move the file position forward
-        pCache->dwFilePos += CACHE_BUFFER_SIZE;
-        if(pCache->dwFilePos >= pCache->dwFileSize)
-            return 0;
-
-        // Get the number of bytes remaining
-        dwBytesToRead = pCache->dwFileSize - pCache->dwFilePos;
-        if(dwBytesToRead > CACHE_BUFFER_SIZE)
-            dwBytesToRead = CACHE_BUFFER_SIZE;
-
-        // Load the next data chunk to the cache
-        SFileSetFilePointer(pCache->hFile, pCache->dwFilePos, NULL, FILE_BEGIN);
-        SFileReadFile(pCache->hFile, pCache->Buffer, dwBytesToRead, &dwBytesRead, NULL);
-
-        // If we didn't read anything, it might mean that the block
-        // of the file is not available (in case of partial MPQs).
-        // We stop reading the file at this point, because the rest
-        // of the listfile is unreliable
-        if(dwBytesRead == 0)
-            return 0;
-
-        // Set the buffer pointers
-        pCache->pBegin =
-        pCache->pPos = &pCache->Buffer[0];
-        pCache->pEnd = pCache->pBegin + dwBytesRead;
-    }
-
-    return dwBytesRead;
-}
-
-static size_t ReadListFileLine(TListFileCache * pCache, char * szLine, int nMaxChars)
+static char * ReadListFileLine(TListFileCache * pCache, size_t * PtrLength)
 {
-    char * szLineBegin = szLine;
-    char * szLineEnd = szLine + nMaxChars - 1;
-    char * szExtraString = NULL;
+    LPBYTE pbLineBegin;
+    LPBYTE pbLineEnd;
+    LPBYTE pbExtraString = NULL;
     
     // Skip newlines, spaces, tabs and another non-printable stuff
-    for(;;)
-    {
-        // If we need to reload the cache, do it
-        if(pCache->pPos == pCache->pEnd)
-        {
-            if(ReloadListFileCache(pCache) == 0)
-                break;
-        }
-
-        // If we found a non-whitespace character, stop
-        if(*pCache->pPos > 0x20)
-            break;
-
-        // Skip the character
+    while(pCache->pPos < pCache->pEnd && pCache->pPos[0] <= 0x20)
         pCache->pPos++;
-    }
+    
+    // Set the line begin and end
+    if(pCache->pPos >= pCache->pEnd)
+        return NULL;
+    pbLineBegin = pbLineEnd = pCache->pPos;
 
     // Copy the remaining characters
-    while(szLine < szLineEnd)
+    while(pCache->pPos < pCache->pEnd && pCache->pPos[0] != 0x0A && pCache->pPos[0] != 0x0D)
     {
-        // If we need to reload the cache, do it now and resume copying
-        if(pCache->pPos == pCache->pEnd)
-        {
-            if(ReloadListFileCache(pCache) == 0)
-                break;
-        }
-
-        // If we have found a newline, stop loading
-        if(*pCache->pPos == 0x0D || *pCache->pPos == 0x0A)
-            break;
-
         // Blizzard listfiles can also contain information about patch:
         // Pass1\Files\MacOS\unconditional\user\Background Downloader.app\Contents\Info.plist~Patch(Data#frFR#base-frFR,1326)
-        if(*pCache->pPos == '~')
-            szExtraString = szLine;
-
-        // Remember that last occurence of a slash or backslash
-//      if(*pCache->pPos == '\\' || *pCache->pPos == '/')
-//          szPlainName = szLine + 1;
+        if(pCache->pPos[0] == '~')
+            pbExtraString = pCache->pPos;
 
         // Copy the character
-        *szLine++ = *pCache->pPos++;
+        pCache->pPos++;
     }
 
-    // Terminate line with zero
-    *szLine = 0;
-
     // If there was extra string after the file name, clear it
-    if(szExtraString != NULL)
+    if(pbExtraString != NULL)
     {
-        if(szExtraString[0] == '~' && szExtraString[1] == 'P')
+        if(pbExtraString[0] == '~' && pbExtraString[1] == 'P')
         {
-            szLine = szExtraString;
-            *szExtraString = 0;
+            pbLineEnd = pbExtraString;
+            pbLineEnd[0] = 0;
         }
     }
+    else
+    {
+        pbLineEnd = pCache->pPos++;
+        pbLineEnd[0] = 0;
+    }
 
-    // Return the length of the line
-    return (szLine - szLineBegin);
+    // Give the line to the caller
+    if(PtrLength != NULL)
+        PtrLength[0] = (size_t)(pbLineEnd - pbLineBegin);
+    return (char *)pbLineBegin;
 }
 
 static int CompareFileNodes(const void * p1, const void * p2) 
@@ -486,16 +424,21 @@ static int SFileAddArbitraryListFile(
     HANDLE hListFile)
 {
     TListFileCache * pCache = NULL;
-    size_t nLength;
-    char szFileName[MAX_PATH];
 
     // Create the listfile cache for that file
     pCache = CreateListFileCache(hListFile, NULL);
     if(pCache != NULL)
     {
-        // Load the node list. Add the node for every locale in the archive
-        while((nLength = ReadListFileLine(pCache, szFileName, sizeof(szFileName))) > 0)
-            SListFileCreateNodeForAllLocales(ha, szFileName);
+        char * szFileName;
+        size_t nLength = 0;
+
+        // Get the next line
+        while((szFileName = ReadListFileLine(pCache, &nLength)) != NULL)
+        {
+            // Add the line to the MPQ
+            if(nLength != 0)
+                SListFileCreateNodeForAllLocales(ha, szFileName);
+        }
 
         // Delete the cache
         FreeListFileCache(pCache);
@@ -572,6 +515,36 @@ static int SFileAddInternalListFile(
     return nError;
 }
 
+static bool DoListFileSearch(TListFileCache * pCache, SFILE_FIND_DATA * lpFindFileData)
+{
+    // Check for the valid search handle
+    if(pCache != NULL)
+    {
+        char * szFileName;
+        size_t nLength = 0;
+
+        // Get the next line
+        while((szFileName = ReadListFileLine(pCache, &nLength)) != NULL)
+        {
+            // Check search mask
+            if(nLength != 0 && CheckWildCard(szFileName, pCache->szWildCard))
+            {
+                if(nLength >= sizeof(lpFindFileData->cFileName))
+                    nLength = sizeof(lpFindFileData->cFileName);
+
+                memcpy(lpFindFileData->cFileName, szFileName, nLength);
+                lpFindFileData->cFileName[nLength] = 0;
+                return true;
+            }
+        }
+    }
+
+    // No more files
+    memset(lpFindFileData, 0, sizeof(SFILE_FIND_DATA));
+    SetLastError(ERROR_NO_MORE_FILES);
+    return false;
+}
+
 //-----------------------------------------------------------------------------
 // File functions
 
@@ -609,9 +582,7 @@ HANDLE WINAPI SListFileFindFirstFile(HANDLE hMpq, const char * szListFile, const
 {
     TListFileCache * pCache = NULL;
     HANDLE hListFile = NULL;
-    size_t nLength = 0;
     DWORD dwSearchScope = SFILE_OPEN_LOCAL_FILE;
-    int nError = ERROR_SUCCESS;
 
     // Initialize the structure with zeros
     memset(lpFindFileData, 0, sizeof(SFILE_FIND_DATA));
@@ -628,48 +599,15 @@ HANDLE WINAPI SListFileFindFirstFile(HANDLE hMpq, const char * szListFile, const
     // Open the local/internal listfile
     if(SFileOpenFileEx(hMpq, szListFile, dwSearchScope, &hListFile))
     {
-#ifdef _DEBUG
-//      TMPQNameCache * pNameCache = CreateNameCache(hListFile, szMask);
-//      FreeNameCache(pNameCache);
-#endif
-
-        // Load the listfile to cache
         pCache = CreateListFileCache(hListFile, szMask);
-        if(pCache != NULL)
-        {
-            // Iterate through the listfile
-            for(;;)
-            {
-                // Read the (next) line
-                nLength = ReadListFileLine(pCache, lpFindFileData->cFileName, sizeof(lpFindFileData->cFileName));
-                if(nLength == 0)
-                {
-                    nError = ERROR_NO_MORE_FILES;
-                    break;
-                }
-
-                // If some mask entered, check it
-                if(CheckWildCard(lpFindFileData->cFileName, pCache->szMask))
-                    break;                
-            }
-        }
-        else
-        {
-            SFileCloseFile(hListFile);
-            nError = ERROR_FILE_CORRUPT;
-        }
-    }
-    else
-    {
-        nError = GetLastError();
+        SFileCloseFile(hListFile);
     }
 
-    // Cleanup & exit
-    if(nError != ERROR_SUCCESS)
+    if(!DoListFileSearch(pCache, lpFindFileData))
     {
         memset(lpFindFileData, 0, sizeof(SFILE_FIND_DATA));
+        SetLastError(ERROR_NO_MORE_FILES);
         FreeListFileCache(pCache);
-        SetLastError(nError);
         pCache = NULL;
     }
 
@@ -679,46 +617,13 @@ HANDLE WINAPI SListFileFindFirstFile(HANDLE hMpq, const char * szListFile, const
 
 bool WINAPI SListFileFindNextFile(HANDLE hFind, SFILE_FIND_DATA * lpFindFileData)
 {
-    TListFileCache * pCache = (TListFileCache *)hFind;
-    size_t nLength;
-    int nError = ERROR_INVALID_PARAMETER;
-
-    // Check for parameters
-    if(pCache != NULL)
-    {
-        for(;;)
-        {
-            // Read the (next) line
-            nLength = ReadListFileLine(pCache, lpFindFileData->cFileName, sizeof(lpFindFileData->cFileName));
-            if(nLength == 0)
-            {
-                nError = ERROR_NO_MORE_FILES;
-                break;
-            }
-
-            // If some mask entered, check it
-            if(CheckWildCard(lpFindFileData->cFileName, pCache->szMask))
-            {
-                nError = ERROR_SUCCESS;
-                break;
-            }
-        }
-    }
-
-    if(nError != ERROR_SUCCESS)
-        SetLastError(nError);
-    return (nError == ERROR_SUCCESS);
+    return DoListFileSearch((TListFileCache *)hFind, lpFindFileData);
 }
 
 bool WINAPI SListFileFindClose(HANDLE hFind)
 {
     TListFileCache * pCache = (TListFileCache *)hFind;
 
-    if(pCache == NULL)
-        return false;
-
-    if(pCache->hFile != NULL)
-        SFileCloseFile(pCache->hFile);
     return FreeListFileCache(pCache);
 }
 
diff --git a/test/StormTest.cpp b/test/StormTest.cpp
index 401752a..06e599f 100644
--- a/test/StormTest.cpp
+++ b/test/StormTest.cpp
@@ -4025,11 +4025,11 @@ int main(int argc, char * argv[])
     // Open a stream, paired with remote master (takes hell lot of time!)
 //  if(nError == ERROR_SUCCESS)
 //      nError = TestReadFile_MasterMirror("MPQ_2013_v4_alternate-downloaded.MPQ", "http://www.zezula.net\\mpqs\\alternate.zip", false);
-
+*/
     // Search in listfile
     if(nError == ERROR_SUCCESS)
         nError = TestSearchListFile("ListFile_Blizzard.txt");
-
+/*
     // Test opening local file with SFileOpenFileEx
     if(nError == ERROR_SUCCESS)
         nError = TestOpenLocalFile("ListFile_Blizzard.txt");