/*****************************************************************************/ /* SListFile.cpp Copyright (c) Ladislav Zezula 2004 */ /*---------------------------------------------------------------------------*/ /* Description: */ /*---------------------------------------------------------------------------*/ /* Date Ver Who Comment */ /* -------- ---- --- ------- */ /* 12.06.04 1.00 Lad The first version of SListFile.cpp */ /*****************************************************************************/ #define __STORMLIB_SELF__ #include "StormLib.h" #include "StormCommon.h" #include //----------------------------------------------------------------------------- // Listfile entry structure #define CACHE_BUFFER_SIZE 0x1000 // Size of the cache buffer #define MAX_LISTFILE_SIZE 0x04000000 // Maximum accepted listfile size is about 68 MB struct TListFileCache { char * szWildCard; // Self-relative pointer to file mask LPBYTE pBegin; // The begin of the listfile cache LPBYTE pPos; // Current position in the cache LPBYTE pEnd; // The last character in the file cache // char szWildCard[wildcard_length]; // Followed by the name mask (if any) // char szListFile[listfile_length]; // Followed by the listfile (if any) }; //----------------------------------------------------------------------------- // Local functions (cache) static char * CopyListLine(char * szListLine, const char * szFileName) { // Copy the string while(szFileName[0] != 0) *szListLine++ = *szFileName++; // Append the end-of-line *szListLine++ = 0x0D; *szListLine++ = 0x0A; return szListLine; } static bool FreeListFileCache(TListFileCache * pCache) { // Valid parameter check if(pCache != NULL) STORM_FREE(pCache); return true; } static TListFileCache * CreateListFileCache(HANDLE hListFile, const char * szWildCard) { TListFileCache * pCache = NULL; size_t cchWildCard = 0; DWORD dwBytesRead = 0; DWORD dwFileSize; // Get the amount of bytes that need to be allocated dwFileSize = SFileGetFileSize(hListFile, NULL); if(dwFileSize == 0 || dwFileSize > MAX_LISTFILE_SIZE) return NULL; // Append buffer for name mask, if any if(szWildCard != NULL) cchWildCard = strlen(szWildCard) + 1; // Allocate cache for one file block pCache = (TListFileCache *)STORM_ALLOC(BYTE, sizeof(TListFileCache) + cchWildCard + dwFileSize + 1); if(pCache != NULL) { // Clear the entire structure memset(pCache, 0, sizeof(TListFileCache) + cchWildCard); // Shall we copy the mask? if(cchWildCard != 0) { pCache->szWildCard = (char *)(pCache + 1); memcpy(pCache->szWildCard, szWildCard, cchWildCard); } // Fill-in the rest of the cache pointers pCache->pBegin = (LPBYTE)(pCache + 1) + cchWildCard; // Load the entire listfile to the cache SFileReadFile(hListFile, pCache->pBegin, dwFileSize, &dwBytesRead, NULL); if(dwBytesRead != 0) { // Allocate pointers pCache->pPos = pCache->pBegin; pCache->pEnd = pCache->pBegin + dwBytesRead; } else { FreeListFileCache(pCache); pCache = NULL; } } // Return the cache return pCache; } #ifdef _DEBUG /* TMPQNameCache * CreateNameCache(HANDLE hListFile, const char * szSearchMask) { TMPQNameCache * pNameCache; char * szCachePointer; size_t cbToAllocate; size_t nMaskLength = 1; DWORD dwBytesRead = 0; DWORD dwFileSize; // Get the size of the listfile. Ignore zero or too long ones dwFileSize = SFileGetFileSize(hListFile, NULL); if(dwFileSize == 0 || dwFileSize > MAX_LISTFILE_SIZE) return NULL; // Get the length of the search mask if(szSearchMask == NULL) szSearchMask = "*"; nMaskLength = strlen(szSearchMask) + 1; // Allocate the name cache cbToAllocate = sizeof(TMPQNameCache) + nMaskLength + dwFileSize + 1; pNameCache = (TMPQNameCache *)STORM_ALLOC(BYTE, cbToAllocate); if(pNameCache != NULL) { // Initialize the name cache memset(pNameCache, 0, sizeof(TMPQNameCache)); pNameCache->TotalCacheSize = (DWORD)(nMaskLength + dwFileSize + 1); szCachePointer = (char *)(pNameCache + 1); // Copy the search mask, if any memcpy(szCachePointer, szSearchMask, nMaskLength); pNameCache->FirstNameOffset = (DWORD)nMaskLength; pNameCache->FreeSpaceOffset = (DWORD)nMaskLength; // Read the listfile itself SFileSetFilePointer(hListFile, 0, NULL, FILE_BEGIN); SFileReadFile(hListFile, szCachePointer + nMaskLength, dwFileSize, &dwBytesRead, NULL); // If nothing has been read from the listfile, clear the cache if(dwBytesRead == 0) { STORM_FREE(pNameCache); return NULL; } // Move the free space offset pNameCache->FreeSpaceOffset = pNameCache->FirstNameOffset + dwBytesRead + 1; szCachePointer[nMaskLength + dwBytesRead] = 0; } return pNameCache; } static void FreeNameCache(TMPQNameCache * pNameCache) { if(pNameCache != NULL) STORM_FREE(pNameCache); pNameCache = NULL; } */ #endif // _DEBUG static char * ReadListFileLine(TListFileCache * pCache, size_t * PtrLength) { LPBYTE pbLineBegin; LPBYTE pbLineEnd; LPBYTE pbExtraString = NULL; // Skip newlines, spaces, tabs and another non-printable stuff while(pCache->pPos < pCache->pEnd && pCache->pPos[0] <= 0x20) pCache->pPos++; // Set the line begin and end if(pCache->pPos >= pCache->pEnd) return NULL; pbLineBegin = pbLineEnd = pCache->pPos; // Copy the remaining characters while(pCache->pPos < pCache->pEnd && pCache->pPos[0] != 0x0A && pCache->pPos[0] != 0x0D) { // Blizzard listfiles can also contain information about patch: // Pass1\Files\MacOS\unconditional\user\Background Downloader.app\Contents\Info.plist~Patch(Data#frFR#base-frFR,1326) if(pCache->pPos[0] == '~') pbExtraString = pCache->pPos; // Copy the character pCache->pPos++; } // If there was extra string after the file name, clear it if(pbExtraString != NULL) { if(pbExtraString[0] == '~' && pbExtraString[1] == 'P') { pbLineEnd = pbExtraString; pbLineEnd[0] = 0; } } else { pbLineEnd = pCache->pPos++; pbLineEnd[0] = 0; } // Give the line to the caller if(PtrLength != NULL) PtrLength[0] = (size_t)(pbLineEnd - pbLineBegin); return (char *)pbLineBegin; } static int CompareFileNodes(const void * p1, const void * p2) { char * szFileName1 = *(char **)p1; char * szFileName2 = *(char **)p2; return _stricmp(szFileName1, szFileName2); } static LPBYTE CreateListFile(TMPQArchive * ha, DWORD * pcbListFile) { TFileEntry * pFileTableEnd = ha->pFileTable + ha->dwFileTableSize; TFileEntry * pFileEntry; char ** SortTable = NULL; char * szListFile = NULL; char * szListLine; size_t nFileNodes = 0; size_t cbListFile = 0; size_t nIndex0; size_t nIndex1; // Allocate the table for sorting listfile SortTable = STORM_ALLOC(char*, ha->dwFileTableSize); if(SortTable == NULL) return NULL; // Construct the sort table // Note: in MPQs with multiple locale versions of the same file, // this code causes adding multiple listfile entries. // They will get removed after the listfile sorting for(pFileEntry = ha->pFileTable; pFileEntry < pFileTableEnd; pFileEntry++) { // Only take existing items if((pFileEntry->dwFlags & MPQ_FILE_EXISTS) && pFileEntry->szFileName != NULL) { // Ignore pseudo-names and internal names if(!IsPseudoFileName(pFileEntry->szFileName, NULL) && !IsInternalMpqFileName(pFileEntry->szFileName)) { SortTable[nFileNodes++] = pFileEntry->szFileName; } } } // Remove duplicities if(nFileNodes > 0) { // Sort the table qsort(SortTable, nFileNodes, sizeof(char *), CompareFileNodes); // Count the 0-th item cbListFile += strlen(SortTable[0]) + 2; // Walk through the items and only use the ones that are not duplicated for(nIndex0 = 0, nIndex1 = 1; nIndex1 < nFileNodes; nIndex1++) { // If the next file node is different, we will include it to the result listfile if(_stricmp(SortTable[nIndex1], SortTable[nIndex0]) != 0) { cbListFile += strlen(SortTable[nIndex1]) + 2; nIndex0 = nIndex1; } } // Now allocate buffer for the entire listfile szListFile = szListLine = STORM_ALLOC(char, cbListFile + 1); if(szListFile != NULL) { // Copy the 0-th item szListLine = CopyListLine(szListLine, SortTable[0]); // Walk through the items and only use the ones that are not duplicated for(nIndex0 = 0, nIndex1 = 1; nIndex1 < nFileNodes; nIndex1++) { // If the next file node is different, we will include it to the result listfile if(_stricmp(SortTable[nIndex1], SortTable[nIndex0]) != 0) { // Copy the listfile line szListLine = CopyListLine(szListLine, SortTable[nIndex1]); nIndex0 = nIndex1; } } // Sanity check - does the size match? assert((size_t)(szListLine - szListFile) == cbListFile); } } else { szListFile = STORM_ALLOC(char, 1); cbListFile = 0; } // Free the sort table STORM_FREE(SortTable); // Give away the listfile if(pcbListFile != NULL) *pcbListFile = (DWORD)cbListFile; return (LPBYTE)szListFile; } //----------------------------------------------------------------------------- // Local functions (listfile nodes) // Adds a name into the list of all names. For each locale in the MPQ, // one entry will be created // If the file name is already there, does nothing. static int SListFileCreateNodeForAllLocales(TMPQArchive * ha, const char * szFileName) { TFileEntry * pFileEntry; TMPQHash * pFirstHash; TMPQHash * pHash; // If we have HET table, use that one if(ha->pHetTable != NULL) { pFileEntry = GetFileEntryLocale(ha, szFileName, 0); if(pFileEntry != NULL) { // Allocate file name for the file entry AllocateFileName(ha, pFileEntry, szFileName); } return ERROR_SUCCESS; } // If we have hash table, we use it if(ha->pHashTable != NULL) { // Go while we found something pFirstHash = pHash = GetFirstHashEntry(ha, szFileName); while(pHash != NULL) { // Allocate file name for the file entry AllocateFileName(ha, ha->pFileTable + MPQ_BLOCK_INDEX(pHash), szFileName); // Now find the next language version of the file pHash = GetNextHashEntry(ha, pFirstHash, pHash); } return ERROR_SUCCESS; } return ERROR_CAN_NOT_COMPLETE; } // Saves the whole listfile to the MPQ int SListFileSaveToMpq(TMPQArchive * ha) { TMPQFile * hf = NULL; LPBYTE pbListFile; DWORD cbListFile = 0; int nError = ERROR_SUCCESS; // Only save the listfile if we should do so if(ha->dwFileFlags1 != 0) { // At this point, we expect to have at least one reserved entry in the file table assert(ha->dwFlags & MPQ_FLAG_LISTFILE_NEW); assert(ha->dwReservedFiles > 0); // Create the raw data that is to be written to (listfile) // Note: Creating the raw data before the (listfile) has been created in the MPQ // causes that the name of the listfile will not be included in the listfile itself. // That is OK, because (listfile) in Blizzard MPQs does not contain it either. pbListFile = CreateListFile(ha, &cbListFile); if(pbListFile != NULL) { // Determine the real flags for (listfile) if(ha->dwFileFlags1 == MPQ_FILE_EXISTS) ha->dwFileFlags1 = GetDefaultSpecialFileFlags(cbListFile, ha->pHeader->wFormatVersion); // Create the listfile in the MPQ nError = SFileAddFile_Init(ha, LISTFILE_NAME, 0, cbListFile, LANG_NEUTRAL, ha->dwFileFlags1 | MPQ_FILE_REPLACEEXISTING, &hf); // Write the listfile raw data to it if(nError == ERROR_SUCCESS) { // Write the content of the listfile to the MPQ nError = SFileAddFile_Write(hf, pbListFile, cbListFile, MPQ_COMPRESSION_ZLIB); SFileAddFile_Finish(hf); } // Clear the listfile flags ha->dwFlags &= ~(MPQ_FLAG_LISTFILE_NEW | MPQ_FLAG_LISTFILE_NONE); ha->dwReservedFiles--; // Free the listfile buffer STORM_FREE(pbListFile); } else { // If the (listfile) file would be empty, its OK nError = (cbListFile == 0) ? ERROR_SUCCESS : ERROR_NOT_ENOUGH_MEMORY; } } return nError; } static int SFileAddArbitraryListFile( TMPQArchive * ha, HANDLE hListFile) { TListFileCache * pCache = NULL; // Create the listfile cache for that file pCache = CreateListFileCache(hListFile, NULL); if(pCache != NULL) { char * szFileName; size_t nLength = 0; // Get the next line while((szFileName = ReadListFileLine(pCache, &nLength)) != NULL) { // Add the line to the MPQ if(nLength != 0) SListFileCreateNodeForAllLocales(ha, szFileName); } // Delete the cache FreeListFileCache(pCache); } return (pCache != NULL) ? ERROR_SUCCESS : ERROR_FILE_CORRUPT; } static int SFileAddExternalListFile( TMPQArchive * ha, HANDLE hMpq, const char * szListFile) { HANDLE hListFile; int nError = ERROR_SUCCESS; // Open the external list file if(!SFileOpenFileEx(hMpq, szListFile, SFILE_OPEN_LOCAL_FILE, &hListFile)) return GetLastError(); // Add the data from the listfile to MPQ nError = SFileAddArbitraryListFile(ha, hListFile); SFileCloseFile(hListFile); return nError; } static int SFileAddInternalListFile( TMPQArchive * ha, HANDLE hMpq) { TMPQHash * pFirstHash; TMPQHash * pHash; HANDLE hListFile; DWORD dwFileSize; LCID lcSaveLocale = lcFileLocale; bool bIgnoreListFile = false; int nError = ERROR_SUCCESS; // If there is hash table, we need to support multiple listfiles // with different locales (BrooDat.mpq) if(ha->pHashTable != NULL) { pFirstHash = pHash = GetFirstHashEntry(ha, LISTFILE_NAME); while(nError == ERROR_SUCCESS && pHash != NULL) { // Set the prefered locale to that from list file SFileSetLocale(pHash->lcLocale); // Attempt to open the file with that locale if(SFileOpenFileEx(hMpq, LISTFILE_NAME, 0, &hListFile)) { // If the archive is a malformed map, ignore too large listfiles if(ha->dwFlags & MPQ_FLAG_MALFORMED) { dwFileSize = SFileGetFileSize(hListFile, NULL); bIgnoreListFile = (dwFileSize > 0x40000); } // Add the data from the listfile to MPQ if(bIgnoreListFile == false) nError = SFileAddArbitraryListFile(ha, hListFile); SFileCloseFile(hListFile); } // Restore the original locale SFileSetLocale(lcSaveLocale); // Move to the next hash pHash = GetNextHashEntry(ha, pFirstHash, pHash); } } else { // Open the external list file if(SFileOpenFileEx(hMpq, LISTFILE_NAME, 0, &hListFile)) { // Add the data from the listfile to MPQ // The function also closes the listfile handle nError = SFileAddArbitraryListFile(ha, hListFile); SFileCloseFile(hListFile); } } // Return the result of the operation return nError; } static bool DoListFileSearch(TListFileCache * pCache, SFILE_FIND_DATA * lpFindFileData) { // Check for the valid search handle if(pCache != NULL) { char * szFileName; size_t nLength = 0; // Get the next line while((szFileName = ReadListFileLine(pCache, &nLength)) != NULL) { // Check search mask if(nLength != 0 && CheckWildCard(szFileName, pCache->szWildCard)) { if(nLength >= sizeof(lpFindFileData->cFileName)) nLength = sizeof(lpFindFileData->cFileName) - 1; memcpy(lpFindFileData->cFileName, szFileName, nLength); lpFindFileData->cFileName[nLength] = 0; return true; } } } // No more files memset(lpFindFileData, 0, sizeof(SFILE_FIND_DATA)); SetLastError(ERROR_NO_MORE_FILES); return false; } //----------------------------------------------------------------------------- // File functions // Adds a listfile into the MPQ archive. int WINAPI SFileAddListFile(HANDLE hMpq, const char * szListFile) { TMPQArchive * ha = (TMPQArchive *)hMpq; int nError = ERROR_SUCCESS; // Add the listfile for each MPQ in the patch chain while(ha != NULL) { if(szListFile != NULL) nError = SFileAddExternalListFile(ha, hMpq, szListFile); else nError = SFileAddInternalListFile(ha, hMpq); // Also, add three special files to the listfile: // (listfile) itself, (attributes) and (signature) SListFileCreateNodeForAllLocales(ha, LISTFILE_NAME); SListFileCreateNodeForAllLocales(ha, SIGNATURE_NAME); SListFileCreateNodeForAllLocales(ha, ATTRIBUTES_NAME); // Move to the next archive in the chain ha = ha->haPatch; } return nError; } //----------------------------------------------------------------------------- // Enumerating files in listfile HANDLE WINAPI SListFileFindFirstFile(HANDLE hMpq, const char * szListFile, const char * szMask, SFILE_FIND_DATA * lpFindFileData) { TListFileCache * pCache = NULL; HANDLE hListFile = NULL; DWORD dwSearchScope = SFILE_OPEN_LOCAL_FILE; // Initialize the structure with zeros memset(lpFindFileData, 0, sizeof(SFILE_FIND_DATA)); // If the szListFile is NULL, it means we have to open internal listfile if(szListFile == NULL) { // Use SFILE_OPEN_ANY_LOCALE for listfile. This will allow us to load // the listfile even if there is only non-neutral version of the listfile in the MPQ dwSearchScope = SFILE_OPEN_ANY_LOCALE; szListFile = LISTFILE_NAME; } // Open the local/internal listfile if(SFileOpenFileEx(hMpq, szListFile, dwSearchScope, &hListFile)) { pCache = CreateListFileCache(hListFile, szMask); SFileCloseFile(hListFile); } if(!DoListFileSearch(pCache, lpFindFileData)) { memset(lpFindFileData, 0, sizeof(SFILE_FIND_DATA)); SetLastError(ERROR_NO_MORE_FILES); FreeListFileCache(pCache); pCache = NULL; } // Return the listfile cache as handle return (HANDLE)pCache; } bool WINAPI SListFileFindNextFile(HANDLE hFind, SFILE_FIND_DATA * lpFindFileData) { return DoListFileSearch((TListFileCache *)hFind, lpFindFileData); } bool WINAPI SListFileFindClose(HANDLE hFind) { TListFileCache * pCache = (TListFileCache *)hFind; return FreeListFileCache(pCache); }