diff options
| -rw-r--r-- | src/FileStream.cpp | 801 | ||||
| -rw-r--r-- | src/FileStream.h | 18 | ||||
| -rw-r--r-- | src/SBaseFileTable.cpp | 95 | ||||
| -rw-r--r-- | src/SFileAddFile.cpp | 2 | ||||
| -rw-r--r-- | src/SFileAttributes.cpp | 45 | ||||
| -rw-r--r-- | src/SFileCompactArchive.cpp | 2 | ||||
| -rw-r--r-- | src/SFileCreateArchive.cpp | 6 | ||||
| -rw-r--r-- | src/SFileExtractFile.cpp | 2 | ||||
| -rw-r--r-- | src/SFileGetFileInfo.cpp | 18 | ||||
| -rw-r--r-- | src/SFileOpenArchive.cpp | 31 | ||||
| -rw-r--r-- | src/SFileOpenFileEx.cpp | 2 | ||||
| -rw-r--r-- | src/SFilePatchArchives.cpp | 2 | ||||
| -rw-r--r-- | src/StormCommon.h | 2 | ||||
| -rw-r--r-- | src/StormLib.h | 38 | ||||
| -rw-r--r-- | test/TLogHelper.cpp | 6 | ||||
| -rw-r--r-- | test/Test.cpp | 370 |
16 files changed, 1051 insertions, 389 deletions
diff --git a/src/FileStream.cpp b/src/FileStream.cpp index 58543e3..c196f11 100644 --- a/src/FileStream.cpp +++ b/src/FileStream.cpp @@ -97,7 +97,7 @@ static bool BaseFile_Create(TFileStream * pStream) return true; } -static bool BaseFile_Open(TFileStream * pStream, DWORD dwStreamFlags) +static bool BaseFile_Open(TFileStream * pStream, const TCHAR * szFileName, DWORD dwStreamFlags) { #ifdef PLATFORM_WINDOWS { @@ -106,7 +106,7 @@ static bool BaseFile_Open(TFileStream * pStream, DWORD dwStreamFlags) DWORD dwWriteShare = (dwStreamFlags & STREAM_FLAG_WRITE_SHARE) ? FILE_SHARE_WRITE : 0; // Open the file - pStream->Base.File.hFile = CreateFile(pStream->szFileName, + pStream->Base.File.hFile = CreateFile(szFileName, FILE_READ_DATA | FILE_READ_ATTRIBUTES | dwWriteAccess, FILE_SHARE_READ | dwWriteShare, NULL, @@ -417,7 +417,7 @@ static void BaseFile_Init(TFileStream * pStream) //----------------------------------------------------------------------------- // Local functions - base memory-mapped file support -static bool BaseMap_Open(TFileStream * pStream, DWORD dwStreamFlags) +static bool BaseMap_Open(TFileStream * pStream, const TCHAR * szFileName, DWORD dwStreamFlags) { #ifdef PLATFORM_WINDOWS @@ -430,7 +430,7 @@ static bool BaseMap_Open(TFileStream * pStream, DWORD dwStreamFlags) dwStreamFlags = dwStreamFlags; // Open the file for read access - hFile = CreateFile(pStream->szFileName, FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); + hFile = CreateFile(szFileName, FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); if(hFile != NULL) { // Retrieve file size. Don't allow mapping file of a zero size. @@ -588,11 +588,10 @@ static const TCHAR * BaseHttp_ExtractServerName(const TCHAR * szFileName, TCHAR return szFileName; } -static bool BaseHttp_Open(TFileStream * pStream, DWORD dwStreamFlags) +static bool BaseHttp_Open(TFileStream * pStream, const TCHAR * szFileName, DWORD dwStreamFlags) { #ifdef PLATFORM_WINDOWS - const TCHAR * szFileName; HINTERNET hRequest; DWORD dwTemp = 0; bool bFileAvailable = false; @@ -624,7 +623,7 @@ static bool BaseHttp_Open(TFileStream * pStream, DWORD dwStreamFlags) DWORD dwFlags = INTERNET_FLAG_KEEP_CONNECTION | INTERNET_FLAG_NO_UI | INTERNET_FLAG_NO_CACHE_WRITE; // Initiate connection with the server - szFileName = BaseHttp_ExtractServerName(pStream->szFileName, szServerName); + szFileName = BaseHttp_ExtractServerName(szFileName, szServerName); pStream->Base.Http.hConnect = InternetConnect(pStream->Base.Http.hInternet, szServerName, INTERNET_DEFAULT_HTTP_PORT, @@ -821,11 +820,11 @@ static bool BlockStream_Read( DWORD BlockSize = pStream->BlockSize; DWORD BlockCount; bool bPrevBlockAvailable; + bool bCallbackCalled = false; bool bBlockAvailable; bool bResult = true; // The base block read function must be present - assert(pStream->BlockCheck != NULL); assert(pStream->BlockRead != NULL); // NOP reading of zero bytes @@ -837,7 +836,7 @@ static bool BlockStream_Read( EndOffset = ByteOffset + dwBytesToRead; if(EndOffset > pStream->StreamSize) { - SetLastError(ERROR_CAN_NOT_COMPLETE); + SetLastError(ERROR_HANDLE_EOF); return false; } @@ -863,6 +862,7 @@ static bool BlockStream_Read( { // Now parse the blocks and send the block read request // to all blocks with the same availability + assert(pStream->BlockCheck != NULL); bPrevBlockAvailable = pStream->BlockCheck(pStream, BlockOffset); // Loop as long as we have something to read @@ -874,6 +874,13 @@ static bool BlockStream_Read( // If the availability has changed, read all blocks up to this one if(bBlockAvailable != bPrevBlockAvailable) { + // Call the file stream callback, if the block is not available + if(pStream->pMaster && pStream->pfnCallback && bPrevBlockAvailable == false) + { + pStream->pfnCallback(pStream->UserData, BlockOffset0, (DWORD)(BlockOffset - BlockOffset0)); + bCallbackCalled = true; + } + // Load the continuous blocks with the same availability assert(BlockOffset > BlockOffset0); bResult = pStream->BlockRead(pStream, BlockOffset0, BlockOffset, BlockBuffer, BytesNeeded, bPrevBlockAvailable); @@ -894,6 +901,13 @@ static bool BlockStream_Read( // If there is a block(s) remaining to be read, do it if(BlockOffset > BlockOffset0) { + // Call the file stream callback, if the block is not available + if(pStream->pMaster && pStream->pfnCallback && bPrevBlockAvailable == false) + { + pStream->pfnCallback(pStream->UserData, BlockOffset0, (DWORD)(BlockOffset - BlockOffset0)); + bCallbackCalled = true; + } + // Read the complete blocks from the file if(BlockOffset > pStream->StreamSize) BlockOffset = pStream->StreamSize; @@ -914,6 +928,15 @@ static bool BlockStream_Read( memcpy(pvBuffer, TransferBuffer + BlockBufferOffset, dwBytesToRead); pStream->StreamPos = ByteOffset + dwBytesToRead; } + else + { + // If the block read failed, set the last error + SetLastError(ERROR_FILE_INCOMPLETE); + } + + // Call the callback to indicate we are done + if(bCallbackCalled) + pStream->pfnCallback(pStream->UserData, 0, 0); // Free the block buffer and return STORM_FREE(TransferBuffer); @@ -955,7 +978,7 @@ static STREAM_INIT StreamBaseInit[4] = }; // This function allocates an empty structure for the file stream -// The stream structure is created as variable length, linear block of data +// The stream structure is created as flat block, variable length // The file name is placed after the end of the stream structure data static TFileStream * AllocateFileStream( const TCHAR * szFileName, @@ -1015,9 +1038,9 @@ static TFileStream * AllocateFileStream( } //----------------------------------------------------------------------------- -// Local functions - linear stream support +// Local functions - flat stream support -static DWORD LinearStream_CheckFile(TBlockStream * pStream) +static DWORD FlatStream_CheckFile(TBlockStream * pStream) { LPBYTE FileBitmap = (LPBYTE)pStream->FileBitmap; DWORD WholeByteCount = (pStream->BlockCount / 8); @@ -1043,7 +1066,7 @@ static DWORD LinearStream_CheckFile(TBlockStream * pStream) return 1; } -static bool LinearStream_LoadBitmap(TBlockStream * pStream) +static bool FlatStream_LoadBitmap(TBlockStream * pStream) { FILE_BITMAP_FOOTER Footer; ULONGLONG ByteOffset; @@ -1060,7 +1083,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) { // Load the bitmap footer ByteOffset = pStream->Base.File.FileSize - sizeof(FILE_BITMAP_FOOTER); - if(pStream->StreamRead(pStream, &ByteOffset, &Footer, sizeof(FILE_BITMAP_FOOTER))) + if(pStream->BaseRead(pStream, &ByteOffset, &Footer, sizeof(FILE_BITMAP_FOOTER))) { // Make sure that the array is properly BSWAP-ed BSWAP_ARRAY32_UNSIGNED((LPDWORD)(&Footer), sizeof(FILE_BITMAP_FOOTER)); @@ -1076,7 +1099,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) // Check if the sizes match if(ByteOffset + BitmapSize + sizeof(FILE_BITMAP_FOOTER) == pStream->Base.File.FileSize) { - // Allocate space for the linear bitmap + // Allocate space for the bitmap FileBitmap = STORM_ALLOC(BYTE, BitmapSize); if(FileBitmap != NULL) { @@ -1095,7 +1118,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) pStream->BitmapSize = BitmapSize; pStream->BlockSize = Footer.BlockSize; pStream->BlockCount = BlockCount; - pStream->IsComplete = LinearStream_CheckFile(pStream); + pStream->IsComplete = FlatStream_CheckFile(pStream); return true; } } @@ -1106,7 +1129,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) return false; } -static void LinearStream_UpdateBitmap( +static void FlatStream_UpdateBitmap( TBlockStream * pStream, // Pointer to an open stream ULONGLONG StartOffset, ULONGLONG EndOffset) @@ -1137,9 +1160,12 @@ static void LinearStream_UpdateBitmap( ByteIndex += (BitMask >> 0x07); BitMask = (BitMask >> 0x07) | (BitMask << 0x01); } + + // Increment the bitmap update count + pStream->IsModified = 1; } -static bool LinearStream_BlockCheck( +static bool FlatStream_BlockCheck( TBlockStream * pStream, // Pointer to an open stream ULONGLONG BlockOffset) { @@ -1159,7 +1185,7 @@ static bool LinearStream_BlockCheck( return (FileBitmap[BlockIndex / 0x08] & BitMask) ? true : false; } -static bool LinearStream_BlockRead( +static bool FlatStream_BlockRead( TBlockStream * pStream, // Pointer to an open stream ULONGLONG StartOffset, ULONGLONG EndOffset, @@ -1170,7 +1196,7 @@ static bool LinearStream_BlockRead( DWORD BytesToRead = (DWORD)(EndOffset - StartOffset); // The starting offset must be aligned to size of the block - assert(pStream->FileBitmap != NULL && pStream->IsComplete == 0); + assert(pStream->FileBitmap != NULL); assert((StartOffset & (pStream->BlockSize - 1)) == 0); assert(StartOffset < EndOffset); @@ -1190,8 +1216,8 @@ static bool LinearStream_BlockRead( // Store the loaded blocks to the mirror file. // Note that this operation is not required to succeed - if(pStream->BaseWrite(pStream, &StartOffset, BlockBuffer, BytesToRead) && bAvailable == false) - LinearStream_UpdateBitmap(pStream, StartOffset, EndOffset); + if(pStream->BaseWrite(pStream, &StartOffset, BlockBuffer, BytesToRead)) + FlatStream_UpdateBitmap(pStream, StartOffset, EndOffset); return true; } @@ -1203,16 +1229,107 @@ static bool LinearStream_BlockRead( } } -static bool LinearStream_CreateMirror(TBlockStream * pStream) +static void FlatStream_Close(TBlockStream * pStream) +{ + FILE_BITMAP_FOOTER Footer; + + if(pStream->FileBitmap && pStream->IsModified) + { + // Write the file bitmap + pStream->BaseWrite(pStream, &pStream->StreamSize, pStream->FileBitmap, pStream->BitmapSize); + + // Prepare and write the file footer + Footer.Signature = ID_FILE_BITMAP_FOOTER; + Footer.Version = 3; + Footer.BuildNumber = 10958; // BUGBUG: What build number shall we set??? + Footer.MapOffsetLo = (DWORD)(pStream->StreamSize & 0xFFFFFFFF); + Footer.MapOffsetHi = (DWORD)(pStream->StreamSize >> 0x20); + Footer.BlockSize = pStream->BlockSize; + BSWAP_ARRAY32_UNSIGNED(&Footer, sizeof(FILE_BITMAP_FOOTER)); + pStream->BaseWrite(pStream, NULL, &Footer, sizeof(FILE_BITMAP_FOOTER)); + } + + // Close the base class + BlockStream_Close(pStream); +} + +static bool FlatStream_CreateMirror(TBlockStream * pStream) { + ULONGLONG MasterSize = 0; + ULONGLONG MirrorSize = 0; + LPBYTE FileBitmap = NULL; + DWORD dwBitmapSize; + DWORD dwBlockCount; + bool bNeedCreateMirrorStream = true; + bool bNeedResizeMirrorStream = true; + // Do we have master function and base creation function? if(pStream->pMaster == NULL || pStream->BaseCreate == NULL) return false; - return false; + // Retrieve the master file size, block count and bitmap size + FileStream_GetSize(pStream->pMaster, &MasterSize); + dwBlockCount = (DWORD)((MasterSize + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE); + dwBitmapSize = (DWORD)((dwBlockCount + 7) / 8); + + // Setup stream size and position + pStream->StreamSize = MasterSize; + pStream->StreamPos = 0; + + // Open the base stream for write access + if(pStream->BaseOpen(pStream, pStream->szFileName, 0)) + { + // If the file open succeeded, check if the file size matches required size + pStream->BaseGetSize(pStream, &MirrorSize); + if(MirrorSize == MasterSize + dwBitmapSize + sizeof(FILE_BITMAP_FOOTER)) + { + // Attempt to load an existing file bitmap + if(FlatStream_LoadBitmap(pStream)) + return true; + + // We need to create new file bitmap + bNeedResizeMirrorStream = false; + } + + // We need to create mirror stream + bNeedCreateMirrorStream = false; + } + + // Create a new stream, if needed + if(bNeedCreateMirrorStream) + { + if(!pStream->BaseCreate(pStream)) + return false; + } + + // If we need to, then resize the mirror stream + if(bNeedResizeMirrorStream) + { + if(!pStream->BaseResize(pStream, MasterSize + dwBitmapSize + sizeof(FILE_BITMAP_FOOTER))) + return false; + } + + // Allocate the bitmap array + FileBitmap = STORM_ALLOC(BYTE, dwBitmapSize); + if(FileBitmap == NULL) + return false; + + // Initialize the bitmap + memset(FileBitmap, 0, dwBitmapSize); + pStream->FileBitmap = FileBitmap; + pStream->BitmapSize = dwBitmapSize; + pStream->BlockSize = DEFAULT_BLOCK_SIZE; + pStream->BlockCount = dwBlockCount; + pStream->IsComplete = 0; + pStream->IsModified = 1; + + // Note: Don't write the stream bitmap right away. + // Doing so would cause sparse file resize on NTFS, + // which would take long time on larger files. + return true; } -static TFileStream * LinearStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) +static TFileStream * FlatStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) { TBlockStream * pStream; @@ -1221,53 +1338,63 @@ static TFileStream * LinearStream_Open(const TCHAR * szFileName, DWORD dwStreamF if(pStream == NULL) return NULL; - // Attempt to open the base stream. If this fails - // and we have a master stream, we can create new mirror - assert(pStream->BaseOpen != NULL); - if(!pStream->BaseOpen(pStream, dwStreamFlags)) + // Do we have a master stream? + if(pStream->pMaster != NULL) { - // Do we have base create function and master stream? -// if(!LinearStream_CreateMirror(pStream)) + if(!FlatStream_CreateMirror(pStream)) { FileStream_Close(pStream); SetLastError(ERROR_FILE_NOT_FOUND); return NULL; } } + else + { + // Attempt to open the base stream + if(!pStream->BaseOpen(pStream, pStream->szFileName, dwStreamFlags)) + return false; - // Set the stream function as if this was linear file without bitmap - pStream->StreamRead = pStream->BaseRead; - pStream->StreamWrite = pStream->BaseWrite; - pStream->StreamResize = pStream->BaseResize; - pStream->StreamGetSize = pStream->BaseGetSize; - pStream->StreamGetPos = pStream->BaseGetPos; - pStream->StreamClose = pStream->BaseClose; + // Load the bitmap, if required to + if(dwStreamFlags & STREAM_FLAG_USE_BITMAP) + FlatStream_LoadBitmap(pStream); - // Setup the stream size - pStream->StreamSize = pStream->Base.File.FileSize; + // Setup the stream size + if(pStream->FileBitmap == NULL) + pStream->StreamSize = pStream->Base.File.FileSize; + pStream->StreamPos = 0; + } - // If we load the bitmap and find out that it's an incomplete file, - // We set the reading functions which check presence of each file block - if(LinearStream_LoadBitmap(pStream)) + // If we have a stream bitmap, set the reading functions + // which check presence of each file block + if(pStream->FileBitmap != NULL) { // Set the stream position to zero. Stream size is already set assert(pStream->StreamSize != 0); pStream->StreamPos = 0; pStream->dwFlags |= STREAM_FLAG_READ_ONLY; - // Set the reading function - if(pStream->IsComplete == 0) - { - // Supply the reading function - pStream->StreamRead = (STREAM_READ)BlockStream_Read; - pStream->StreamGetSize = BlockStream_GetSize; - pStream->StreamGetPos = BlockStream_GetPos; - pStream->StreamClose = (STREAM_CLOSE)BlockStream_Close; - - // Supply the block functions - pStream->BlockCheck = (BLOCK_CHECK)LinearStream_BlockCheck; - pStream->BlockRead = (BLOCK_READ)LinearStream_BlockRead; - } + // Supply the stream functions + pStream->StreamRead = (STREAM_READ)BlockStream_Read; + pStream->StreamGetSize = BlockStream_GetSize; + pStream->StreamGetPos = BlockStream_GetPos; + pStream->StreamClose = (STREAM_CLOSE)FlatStream_Close; + + // Supply the block functions + pStream->BlockCheck = (BLOCK_CHECK)FlatStream_BlockCheck; + pStream->BlockRead = (BLOCK_READ)FlatStream_BlockRead; + } + else + { + // Reset the file position to zero + pStream->Base.File.FilePos = 0; + + // Set the base functions + pStream->StreamRead = pStream->BaseRead; + pStream->StreamWrite = pStream->BaseWrite; + pStream->StreamResize = pStream->BaseResize; + pStream->StreamGetSize = pStream->BaseGetSize; + pStream->StreamGetPos = pStream->BaseGetPos; + pStream->StreamClose = pStream->BaseClose; } return pStream; @@ -1440,7 +1567,7 @@ static TFileStream * PartialStream_Open(const TCHAR * szFileName, DWORD dwStream // Attempt to open the base stream. If this fails // and we have a master stream, we can create new mirror assert(pStream->BaseOpen != NULL); - if(!pStream->BaseOpen(pStream, dwStreamFlags)) + if(!pStream->BaseOpen(pStream, pStream->szFileName, dwStreamFlags)) { // Do we have base create function and master stream? // if(!PartialStream_CreateMirror(pStream)) @@ -1478,7 +1605,7 @@ static TFileStream * PartialStream_Open(const TCHAR * szFileName, DWORD dwStream } //----------------------------------------------------------------------------- -// Local functions - encrypted stream support +// Local functions - MPQE stream support static const char * szKeyTemplate = "expand 32-byte k000000000000000000000000000000000000000000000000"; @@ -1672,7 +1799,7 @@ static void DecryptFileChunk( } } -static bool EncryptedStream_DetectFileKey(TEncryptedStream * pStream) +static bool MpqeStream_DetectFileKey(TEncryptedStream * pStream) { ULONGLONG ByteOffset = 0; BYTE EncryptedHeader[MPQE_CHUNK_SIZE]; @@ -1712,12 +1839,7 @@ static bool EncryptedStream_DetectFileKey(TEncryptedStream * pStream) return false; } -static bool EncryptedStream_BlockCheck(TEncryptedStream *, ULONGLONG) -{ - return true; -} - -static bool EncryptedStream_BlockRead( +static bool MpqeStream_BlockRead( TEncryptedStream * pStream, ULONGLONG StartOffset, ULONGLONG EndOffset, @@ -1731,6 +1853,7 @@ static bool EncryptedStream_BlockRead( assert(StartOffset < EndOffset); assert(bAvailable != false); BytesNeeded = BytesNeeded; + bAvailable = bAvailable; // Read the file from the stream as-is // Limit the reading to number of blocks really needed @@ -1744,67 +1867,7 @@ static bool EncryptedStream_BlockRead( return true; } -/* -static bool EncryptedStream_Read( - TEncryptedStream * pStream, // Pointer to an open stream - ULONGLONG * pByteOffset, // Pointer to file byte offset. If NULL, it reads from the current position - void * pvBuffer, // Pointer to data to be read - DWORD dwBytesToRead) // Number of bytes to read from the file -{ - ULONGLONG StartOffset; // Offset of the first byte to be read from the file - ULONGLONG ByteOffset; // Offset that the caller wants - ULONGLONG EndOffset; // End offset that is to be read from the file - DWORD dwBytesToAllocate; - DWORD dwBytesToDecrypt; - DWORD dwOffsetInCache; - LPBYTE pbMpqData = NULL; - bool bResult = false; - - // Get the byte offset - ByteOffset = (pByteOffset != NULL) ? *pByteOffset : pStream->StreamPos; - - // Cut it down to MPQE chunk size - StartOffset = ByteOffset; - StartOffset = StartOffset & ~(MPQE_CHUNK_SIZE - 1); - EndOffset = ByteOffset + dwBytesToRead; - - // Calculate number of bytes to decrypt - dwBytesToDecrypt = (DWORD)(EndOffset - StartOffset); - dwBytesToAllocate = (dwBytesToDecrypt + MPQE_CHUNK_SIZE - 1) & ~(MPQE_CHUNK_SIZE - 1); - - // Allocate buffers for encrypted and decrypted data - pbMpqData = STORM_ALLOC(BYTE, dwBytesToAllocate); - if(pbMpqData) - { - // Get the offset of the desired data in the cache - dwOffsetInCache = (DWORD)(ByteOffset - StartOffset); - - // Read the file from the stream as-is - if(pStream->BaseRead(pStream, &StartOffset, pbMpqData, dwBytesToDecrypt)) - { - // Decrypt the data - DecryptFileChunk((LPDWORD)pbMpqData, pStream->Key, StartOffset, dwBytesToAllocate); - - // Copy the decrypted data - memcpy(pvBuffer, pbMpqData + dwOffsetInCache, dwBytesToRead); - bResult = true; - } - else - { - assert(false); - } - - // Free decryption buffer - STORM_FREE(pbMpqData); - } - - // Update stream position - pStream->StreamPos = ByteOffset + dwBytesToRead; - return bResult; -} -*/ - -static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) +static TFileStream * MpqeStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) { TEncryptedStream * pStream; @@ -1815,11 +1878,11 @@ static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStre // Attempt to open the base stream assert(pStream->BaseOpen != NULL); - if(!pStream->BaseOpen(pStream, dwStreamFlags)) + if(!pStream->BaseOpen(pStream, pStream->szFileName, dwStreamFlags)) return NULL; // Determine the encryption key for the MPQ - if(EncryptedStream_DetectFileKey(pStream)) + if(MpqeStream_DetectFileKey(pStream)) { // Set the stream position and size assert(pStream->StreamSize != 0); @@ -1833,8 +1896,7 @@ static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStre pStream->StreamClose = pStream->BaseClose; // Supply the block functions - pStream->BlockCheck = (BLOCK_CHECK)EncryptedStream_BlockCheck; - pStream->BlockRead = (BLOCK_READ)EncryptedStream_BlockRead; + pStream->BlockRead = (BLOCK_READ)MpqeStream_BlockRead; return pStream; } @@ -1845,6 +1907,203 @@ static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStre } //----------------------------------------------------------------------------- +// Local functions - Block4 stream support + +#define BLOCK4_BLOCK_SIZE 0x4000 // Size of one block +#define BLOCK4_HASH_SIZE 0x20 // Size of MD5 hash that is after each block +#define BLOCK4_MAX_BLOCKS 0x00002000 // Maximum amount of blocks per file +#define BLOCK4_MAX_FSIZE 0x08040000 // Max size of one file + +static bool Block4Stream_BlockRead( + TBlockStream * pStream, // Pointer to an open stream + ULONGLONG StartOffset, + ULONGLONG EndOffset, + LPBYTE BlockBuffer, + DWORD BytesNeeded, + bool bAvailable) +{ + TBaseProviderData * BaseArray = (TBaseProviderData *)pStream->FileBitmap; + ULONGLONG ByteOffset; + DWORD BytesToRead; + DWORD StreamIndex; + DWORD BlockIndex; + + // The starting offset must be aligned to size of the block + assert(pStream->FileBitmap != NULL); + assert((StartOffset & (pStream->BlockSize - 1)) == 0); + assert(StartOffset < EndOffset); + assert(bAvailable == true); + + while(BytesNeeded != 0) + { + // Calculate the block index and the file index + StreamIndex = (DWORD)((StartOffset / pStream->BlockSize) / BLOCK4_MAX_BLOCKS); + BlockIndex = (DWORD)((StartOffset / pStream->BlockSize) % BLOCK4_MAX_BLOCKS); + if(StreamIndex > pStream->BitmapSize) + return false; + + // Prepare the appropriate stream to the base variables + memcpy(&pStream->Base, BaseArray + StreamIndex, sizeof(TBaseProviderData)); + + // Calculate the block offset + ByteOffset = ((ULONGLONG)BlockIndex * (BLOCK4_BLOCK_SIZE + BLOCK4_HASH_SIZE)); + BytesToRead = min(BytesNeeded, BLOCK4_BLOCK_SIZE); + + // Read from the base stream + if(!pStream->BaseRead(pStream, &ByteOffset, BlockBuffer, BytesToRead)) + return false; + + // Move pointers + StartOffset += BytesToRead; + BlockBuffer += BytesToRead; + BytesNeeded -= BytesToRead; + } + + return true; +} + + +static void Block4Stream_Close(TBlockStream * pStream) +{ + TBaseProviderData * BaseArray = (TBaseProviderData *)pStream->FileBitmap; + + // If we have a non-zero count of base streams, + // we have to close them all + if(BaseArray != NULL) + { + // Close all base streams + for(DWORD i = 0; i < pStream->BitmapSize; i++) + { + memcpy(&pStream->Base, BaseArray + i, sizeof(TBaseProviderData)); + pStream->BaseClose(pStream); + } + } + + // Free the data map, if any + if(pStream->FileBitmap != NULL) + STORM_FREE(pStream->FileBitmap); + pStream->FileBitmap = NULL; + + // Do not call the BaseClose function, + // we closed all handles already + return; +} + +static TFileStream * Block4Stream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) +{ + TBaseProviderData * NewBaseArray = NULL; + ULONGLONG RemainderBlock; + ULONGLONG BlockCount; + ULONGLONG FileSize; + TBlockStream * pStream; + TCHAR * szNameBuff; + size_t nNameLength; + DWORD dwBaseFiles = 0; + DWORD dwBaseFlags; + + // Create new empty stream + pStream = (TBlockStream *)AllocateFileStream(szFileName, sizeof(TBlockStream), dwStreamFlags); + if(pStream == NULL) + return NULL; + + // Sanity check + assert(pStream->BaseOpen != NULL); + + // Get the length of the file name without numeric suffix + nNameLength = _tcslen(pStream->szFileName); + if(pStream->szFileName[nNameLength - 2] == '.' && pStream->szFileName[nNameLength - 1] == '0') + nNameLength -= 2; + pStream->szFileName[nNameLength] = 0; + + // Supply the stream functions + pStream->StreamRead = (STREAM_READ)BlockStream_Read; + pStream->StreamGetSize = BlockStream_GetSize; + pStream->StreamGetPos = BlockStream_GetPos; + pStream->StreamClose = (STREAM_CLOSE)Block4Stream_Close; + pStream->BlockRead = (BLOCK_READ)Block4Stream_BlockRead; + + // Allocate work space for numeric names + szNameBuff = STORM_ALLOC(TCHAR, nNameLength + 4); + if(szNameBuff != NULL) + { + // Set the base flags + dwBaseFlags = (dwStreamFlags & STREAM_PROVIDERS_MASK) | STREAM_FLAG_READ_ONLY; + + // Go all suffixes from 0 to 30 + for(int nSuffix = 0; nSuffix < 30; nSuffix++) + { + // Open the n-th file + _stprintf(szNameBuff, _T("%s.%u"), pStream->szFileName, nSuffix); + if(!pStream->BaseOpen(pStream, szNameBuff, dwBaseFlags)) + break; + + // If the open succeeded, we re-allocate the base provider array + NewBaseArray = STORM_ALLOC(TBaseProviderData, dwBaseFiles + 1); + if(NewBaseArray == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return false; + } + + // Copy the old base data array to the new base data array + if(pStream->FileBitmap != NULL) + { + memcpy(NewBaseArray, pStream->FileBitmap, sizeof(TBaseProviderData) * dwBaseFiles); + STORM_FREE(pStream->FileBitmap); + } + + // Also copy the opened base array + memcpy(NewBaseArray + dwBaseFiles, &pStream->Base, sizeof(TBaseProviderData)); + pStream->FileBitmap = NewBaseArray; + dwBaseFiles++; + + // Get the size of the base stream + pStream->BaseGetSize(pStream, &FileSize); + assert(FileSize <= BLOCK4_MAX_FSIZE); + RemainderBlock = FileSize % (BLOCK4_BLOCK_SIZE + BLOCK4_HASH_SIZE); + BlockCount = FileSize / (BLOCK4_BLOCK_SIZE + BLOCK4_HASH_SIZE); + + // Increment the stream size and number of blocks + pStream->StreamSize += (BlockCount * BLOCK4_BLOCK_SIZE); + pStream->BlockCount += (DWORD)BlockCount; + + // Is this the last file? + if(FileSize < BLOCK4_MAX_FSIZE) + { + if(RemainderBlock) + { + pStream->StreamSize += (RemainderBlock - BLOCK4_HASH_SIZE); + pStream->BlockCount++; + } + break; + } + } + + // Fill the remainining block stream variables + pStream->BitmapSize = dwBaseFiles; + pStream->BlockSize = BLOCK4_BLOCK_SIZE; + pStream->IsComplete = 1; + pStream->IsModified = 0; + + // Fill the remaining stream variables + pStream->StreamPos = 0; + pStream->dwFlags |= STREAM_FLAG_READ_ONLY; + + STORM_FREE(szNameBuff); + } + + // If we opened something, return success + if(dwBaseFiles == 0) + { + FileStream_Close(pStream); + SetLastError(ERROR_FILE_NOT_FOUND); + pStream = NULL; + } + + return pStream; +} + +//----------------------------------------------------------------------------- // Public functions /** @@ -1870,14 +2129,14 @@ TFileStream * FileStream_CreateFile( { TFileStream * pStream; - // We only support creation of linear, local file - if((dwStreamFlags & (STREAM_PROVIDERS_MASK)) != (STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE)) + // We only support creation of flat, local file + if((dwStreamFlags & (STREAM_PROVIDERS_MASK)) != (STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE)) { SetLastError(ERROR_NOT_SUPPORTED); return NULL; } - // Allocate file stream structure for linear stream + // Allocate file stream structure for flat stream pStream = AllocateFileStream(szFileName, sizeof(TBlockStream), dwStreamFlags); if(pStream != NULL) { @@ -1924,41 +2183,27 @@ TFileStream * FileStream_OpenFile( const TCHAR * szFileName, DWORD dwStreamFlags) { - DWORD dwBaseProvider = dwStreamFlags & BASE_PROVIDER_MASK; - - // The "file:" prefix forces the BASE_PROVIDER_FILE - if(!_tcsicmp(szFileName, _T("file:"))) - { - dwBaseProvider = BASE_PROVIDER_FILE; - szFileName += 5; - } - - // The "map:" prefix forces the BASE_PROVIDER_MAP - if(!_tcsicmp(szFileName, _T("map:"))) - { - dwBaseProvider = BASE_PROVIDER_MAP; - szFileName += 4; - } - - // The "http:" prefix forces the BASE_PROVIDER_HTTP - if(!_tcsicmp(szFileName, _T("http:"))) - { - dwBaseProvider = BASE_PROVIDER_HTTP; - szFileName += 5; - } + DWORD dwProvider = dwStreamFlags & STREAM_PROVIDERS_MASK; + size_t nPrefixLength = FileStream_Prefix(szFileName, &dwProvider); // Re-assemble the stream flags - dwStreamFlags = (dwStreamFlags & STREAM_OPTIONS_MASK) | (dwStreamFlags & STREAM_PROVIDER_MASK) | dwBaseProvider; + dwStreamFlags = (dwStreamFlags & STREAM_OPTIONS_MASK) | dwProvider; + szFileName += nPrefixLength; + + // Perform provider-specific open switch(dwStreamFlags & STREAM_PROVIDER_MASK) { - case STREAM_PROVIDER_LINEAR: - return LinearStream_Open(szFileName, dwStreamFlags); + case STREAM_PROVIDER_FLAT: + return FlatStream_Open(szFileName, dwStreamFlags); case STREAM_PROVIDER_PARTIAL: return PartialStream_Open(szFileName, dwStreamFlags); - case STREAM_PROVIDER_ENCRYPTED: - return EncryptedStream_Open(szFileName, dwStreamFlags); + case STREAM_PROVIDER_MPQE: + return MpqeStream_Open(szFileName, dwStreamFlags); + + case STREAM_PROVIDER_BLOCK4: + return Block4Stream_Open(szFileName, dwStreamFlags); default: SetLastError(ERROR_INVALID_PARAMETER); @@ -1967,6 +2212,215 @@ TFileStream * FileStream_OpenFile( } /** + * Returns the file name of the stream + * + * \a pStream Pointer to an open stream + */ +const TCHAR * FileStream_GetFileName(TFileStream * pStream) +{ + assert(pStream != NULL); + return pStream->szFileName; +} + +/** + * Returns the length of the provider prefix. Returns zero if no prefix + * + * \a szFileName Pointer to a stream name (file, mapped file, URL) + * \a pdwStreamProvider Pointer to a DWORD variable that receives stream provider (STREAM_PROVIDER_XXX) + */ + +size_t FileStream_Prefix(const TCHAR * szFileName, DWORD * pdwProvider) +{ + size_t nPrefixLength1 = 0; + size_t nPrefixLength2 = 0; + DWORD dwProvider = 0; + + if(szFileName != NULL) + { + // + // Determine the stream provider + // + + if(!_tcsnicmp(szFileName, _T("flat-"), 5)) + { + dwProvider |= STREAM_PROVIDER_FLAT; + nPrefixLength1 = 5; + } + + if(!_tcsnicmp(szFileName, _T("part-"), 5)) + { + dwProvider |= STREAM_PROVIDER_PARTIAL; + nPrefixLength1 = 5; + } + + if(!_tcsnicmp(szFileName, _T("mpqe-"), 5)) + { + dwProvider |= STREAM_PROVIDER_MPQE; + nPrefixLength1 = 5; + } + + if(!_tcsnicmp(szFileName, _T("blk4-"), 5)) + { + dwProvider |= STREAM_PROVIDER_BLOCK4; + nPrefixLength1 = 5; + } + + // + // Determine the base provider + // + + if(!_tcsnicmp(szFileName+nPrefixLength1, _T("file:"), 5)) + { + dwProvider |= BASE_PROVIDER_FILE; + nPrefixLength2 = 5; + } + + if(!_tcsnicmp(szFileName+nPrefixLength1, _T("map:"), 4)) + { + dwProvider |= BASE_PROVIDER_MAP; + nPrefixLength2 = 4; + } + + if(!_tcsnicmp(szFileName+nPrefixLength1, _T("http:"), 5)) + { + dwProvider |= BASE_PROVIDER_HTTP; + nPrefixLength2 = 5; + } + + // Only accept stream provider if we recognized the base provider + if(nPrefixLength2 != 0) + { + // It is also allowed to put "//" after the base provider, e.g. "file://", "http://" + if(szFileName[nPrefixLength1+nPrefixLength2] == '/' && szFileName[nPrefixLength1+nPrefixLength2+1] == '/') + nPrefixLength2 += 2; + + if(pdwProvider != NULL) + *pdwProvider = dwProvider; + return nPrefixLength1 + nPrefixLength2; + } + } + + return 0; +} + +/** + * Sets a download callback. Whenever the stream needs to download one or more blocks + * from the server, the callback is called + * + * \a pStream Pointer to an open stream + * \a pfnCallback Pointer to callback function + * \a pvUserData Arbitrary user pointer passed to the download callback + */ + +bool FileStream_SetCallback(TFileStream * pStream, SFILE_DOWNLOAD_CALLBACK pfnCallback, void * pvUserData) +{ + TBlockStream * pBlockStream = (TBlockStream *)pStream; + + if(pStream->BlockRead == NULL) + { + SetLastError(ERROR_NOT_SUPPORTED); + return false; + } + + pBlockStream->pfnCallback = pfnCallback; + pBlockStream->UserData = pvUserData; + return true; +} + +/** + * This function gives the block map. The 'pvBitmap' pointer must point to a buffer + * of at least sizeof(STREAM_BLOCK_MAP) size. It can also have size of the complete + * block map (i.e. sizeof(STREAM_BLOCK_MAP) + BitmapSize). In that case, the function + * also copies the bit-based block map. + * + * \a pStream Pointer to an open stream + * \a pvBitmap Pointer to buffer where the block map will be stored + * \a cbBitmap Length of the buffer, of the block map + * \a cbLengthNeeded Length of the bitmap, in bytes + */ + +bool FileStream_GetBitmap(TFileStream * pStream, void * pvBitmap, DWORD cbBitmap, PDWORD pcbLengthNeeded) +{ + TStreamBitmap * pBitmap = (TStreamBitmap *)pvBitmap; + TBlockStream * pBlockStream = (TBlockStream *)pStream; + ULONGLONG BlockOffset; + LPBYTE Bitmap = (LPBYTE)(pBitmap + 1); + DWORD BitmapSize; + DWORD BlockCount; + DWORD BlockSize; + bool bResult = false; + + // Retrieve the size of one block + if(pStream->BlockCheck != NULL) + { + BlockCount = pBlockStream->BlockCount; + BlockSize = pBlockStream->BlockSize; + } + else + { + BlockCount = (DWORD)((pStream->StreamSize + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE); + BlockSize = DEFAULT_BLOCK_SIZE; + } + + // Fill-in the variables + BitmapSize = (BlockCount + 7) / 8; + + // Give the number of blocks + if(pcbLengthNeeded != NULL) + pcbLengthNeeded[0] = sizeof(TStreamBitmap) + BitmapSize; + + // If the length of the buffer is not enough + if(pvBitmap != NULL && cbBitmap != 0) + { + // Give the STREAM_BLOCK_MAP structure + if(cbBitmap >= sizeof(TStreamBitmap)) + { + pBitmap->StreamSize = pStream->StreamSize; + pBitmap->BitmapSize = BitmapSize; + pBitmap->BlockCount = BlockCount; + pBitmap->BlockSize = BlockSize; + pBitmap->IsComplete = (pStream->BlockCheck != NULL) ? pBlockStream->IsComplete : 1; + bResult = true; + } + + // Give the block bitmap, if enough space + if(cbBitmap >= sizeof(TStreamBitmap) + BitmapSize) + { + // Version with bitmap present + if(pStream->BlockCheck != NULL) + { + DWORD ByteIndex = 0; + BYTE BitMask = 0x01; + + // Initialize the map with zeros + memset(Bitmap, 0, BitmapSize); + + // Fill the map + for(BlockOffset = 0; BlockOffset < pStream->StreamSize; BlockOffset += BlockSize) + { + // Set the bit if the block is present + if(pBlockStream->BlockCheck(pStream, BlockOffset)) + Bitmap[ByteIndex] |= BitMask; + + // Move bit position + ByteIndex += (BitMask >> 0x07); + BitMask = (BitMask >> 0x07) | (BitMask << 0x01); + } + } + else + { + memset(Bitmap, 0xFF, BitmapSize); + } + } + } + + // Set last error value and return + if(bResult == false) + SetLastError(ERROR_INSUFFICIENT_BUFFER); + return bResult; +} + +/** * Reads data from the stream * * - Returns true if the read operation succeeded and all bytes have been read @@ -2094,8 +2548,8 @@ bool FileStream_GetFlags(TFileStream * pStream, LPDWORD pdwStreamFlags) */ bool FileStream_Replace(TFileStream * pStream, TFileStream * pNewStream) { - // Only supported on linear files - if((pStream->dwFlags & STREAM_PROVIDERS_MASK) != (STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE)) + // Only supported on flat files + if((pStream->dwFlags & STREAM_PROVIDERS_MASK) != (STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE)) { SetLastError(ERROR_NOT_SUPPORTED); return false; @@ -2117,7 +2571,7 @@ bool FileStream_Replace(TFileStream * pStream, TFileStream * pNewStream) return false; // Now open the base file again - if(BaseFile_Open(pStream, pStream->dwFlags)) + if(BaseFile_Open(pStream, pStream->szFileName, pStream->dwFlags)) return false; // Cleanup the new stream @@ -2126,27 +2580,6 @@ bool FileStream_Replace(TFileStream * pStream, TFileStream * pNewStream) } /** - * Returns the file name of the stream - * - * \a pStream Pointer to an open stream - */ -const TCHAR * FileStream_GetFileName(TFileStream * pStream) -{ - assert(pStream != NULL); - return pStream->szFileName; -} - -/** - * Returns true if the stream is read-only - * - * \a pStream Pointer to an open stream - */ -bool FileStream_IsReadOnly(TFileStream * pStream) -{ - return (pStream->dwFlags & STREAM_FLAG_READ_ONLY) ? true : false; -} - -/** * This function closes an archive file and frees any data buffers * that have been allocated for stream management. The function must also * support partially allocated structure, i.e. one or more buffers diff --git a/src/FileStream.h b/src/FileStream.h index f660a87..60f41d3 100644 --- a/src/FileStream.h +++ b/src/FileStream.h @@ -24,6 +24,7 @@ typedef bool (*STREAM_CREATE)( typedef bool (*STREAM_OPEN)( struct TFileStream * pStream, // Pointer to an unopened stream + const TCHAR * szFileName, // Pointer to file name to be open DWORD dwStreamFlags // Stream flags ); @@ -57,11 +58,11 @@ typedef bool (*STREAM_GETPOS)( ); typedef void (*STREAM_CLOSE)( - struct TFileStream * pStream + struct TFileStream * pStream // Pointer to an open stream ); typedef bool (*BLOCK_READ)( - struct TFileStream * pStream, // Pointer to an opened block stream + struct TFileStream * pStream, // Pointer to a block-oriented stream ULONGLONG StartOffset, // Byte offset of start of the block array ULONGLONG EndOffset, // End offset (either end of the block or end of the file) LPBYTE BlockBuffer, // Pointer to block-aligned buffer @@ -70,8 +71,12 @@ typedef bool (*BLOCK_READ)( ); typedef DWORD (*BLOCK_CHECK)( - struct TFileStream * pStream, - ULONGLONG BlockOffset + struct TFileStream * pStream, // Pointer to a block-oriented stream + ULONGLONG BlockOffset // Offset of the file to check + ); + +typedef void (*BLOCK_SAVEMAP)( + struct TFileStream * pStream // Pointer to a block-oriented stream ); //----------------------------------------------------------------------------- @@ -187,12 +192,15 @@ struct TFileStream struct TBlockStream : public TFileStream { + SFILE_DOWNLOAD_CALLBACK pfnCallback; // Callback for downloading void * FileBitmap; // Array of bits for file blocks + void * UserData; // User data to be passed to the download callback DWORD BitmapSize; // Size of the file bitmap (in bytes) DWORD BlockSize; // Size of one block, in bytes DWORD BlockCount; // Number of data blocks in the file DWORD IsComplete; // If nonzero, no blocks are missing -}; + DWORD IsModified; // nonzero if the bitmap has been modified +}; //----------------------------------------------------------------------------- // Structure for encrypted stream diff --git a/src/SBaseFileTable.cpp b/src/SBaseFileTable.cpp index fe02931..322211c 100644 --- a/src/SBaseFileTable.cpp +++ b/src/SBaseFileTable.cpp @@ -250,7 +250,7 @@ static DWORD GetMaxFileOffset32(TMPQArchive * ha) return dwMaxFileOffset; } -static ULONGLONG DetermineEndOfArchive_V1_V2( +static ULONGLONG DetermineArchiveSize_V1_V2( TMPQArchive * ha, TMPQHeader * pHeader, ULONGLONG MpqOffset, @@ -264,9 +264,17 @@ static ULONGLONG DetermineEndOfArchive_V1_V2( // Check if we can rely on the archive size in the header if((FileSize >> 0x20) == 0) { - dwArchiveSize32 = (DWORD)(FileSize - MpqOffset); - if(pHeader->dwBlockTablePos < pHeader->dwArchiveSize && pHeader->dwArchiveSize <= dwArchiveSize32) - return pHeader->dwArchiveSize; + if(pHeader->dwBlockTablePos < pHeader->dwArchiveSize) + { + // If the block table end matches the archive size, we trust the archive size + if(pHeader->dwBlockTablePos + (pHeader->dwBlockTableSize * sizeof(TMPQBlock)) == pHeader->dwArchiveSize) + return pHeader->dwArchiveSize; + + // If the archive size in the header is less than real file size + dwArchiveSize32 = (DWORD)(FileSize - MpqOffset); + if(pHeader->dwArchiveSize <= dwArchiveSize32) + return pHeader->dwArchiveSize; + } } // Check if there is a signature header @@ -385,7 +393,7 @@ int ConvertMpqHeaderToFormat4( // Determine the archive size on malformed MPQs if(ha->dwFlags & MPQ_FLAG_MALFORMED) { - pHeader->ArchiveSize64 = DetermineEndOfArchive_V1_V2(ha, pHeader, MpqOffset, FileSize); + pHeader->ArchiveSize64 = DetermineArchiveSize_V1_V2(ha, pHeader, MpqOffset, FileSize); pHeader->dwArchiveSize = (DWORD)pHeader->ArchiveSize64; } break; @@ -423,25 +431,25 @@ int ConvertMpqHeaderToFormat4( if(pHeader->HiBlockTablePos64 != 0) { // BlockTableSize64 may be less than TblSize * sizeof(TMPQBlock). - // That means that the hash table is compressed. + // That means that the hi-block table is compressed. pHeader->BlockTableSize64 = pHeader->HiBlockTablePos64 - BlockTablePos64; assert(pHeader->BlockTableSize64 <= (pHeader->dwBlockTableSize * sizeof(TMPQBlock))); - // Determine the size of the hi-block table - pHeader->HiBlockTableSize64 = DetermineEndOfArchive_V1_V2(ha, pHeader, MpqOffset, FileSize) - pHeader->HiBlockTablePos64; - assert(pHeader->HiBlockTableSize64 == (pHeader->dwBlockTableSize * sizeof(USHORT))); + // Determine real archive size + pHeader->ArchiveSize64 = DetermineArchiveSize_V1_V2(ha, pHeader, MpqOffset, FileSize); - // Recalculate the archive size - pHeader->ArchiveSize64 = pHeader->HiBlockTablePos64 + pHeader->HiBlockTableSize64; + // Calculate the size of the hi-block table + pHeader->HiBlockTableSize64 = pHeader->ArchiveSize64 - pHeader->HiBlockTablePos64; + assert(pHeader->HiBlockTableSize64 == (pHeader->dwBlockTableSize * sizeof(USHORT))); } else { - // Block table size is the end of the archive minus the block table position - pHeader->BlockTableSize64 = DetermineEndOfArchive_V1_V2(ha, pHeader, MpqOffset, FileSize) - BlockTablePos64; - assert(pHeader->BlockTableSize64 <= (pHeader->dwBlockTableSize * sizeof(TMPQBlock))); + // Determine real archive size + pHeader->ArchiveSize64 = DetermineArchiveSize_V1_V2(ha, pHeader, MpqOffset, FileSize); - // dwArchiveSize in the header v 2.0 is 32-bit only - pHeader->ArchiveSize64 = BlockTablePos64 + pHeader->BlockTableSize64; + // Calculate size of the block table + pHeader->BlockTableSize64 = pHeader->ArchiveSize64 - BlockTablePos64; + assert(pHeader->BlockTableSize64 <= (pHeader->dwBlockTableSize * sizeof(TMPQBlock))); } } else @@ -1041,7 +1049,7 @@ static void CreateHetHeader( pHetHeader->dwIndexTableSize; } -TMPQHetTable * CreateHetTable(DWORD dwFileCount, DWORD dwNameHashBitSize, LPBYTE pbSrcData) +TMPQHetTable * CreateHetTable(DWORD dwEntryCount, DWORD dwTotalCount, DWORD dwNameHashBitSize, LPBYTE pbSrcData) { TMPQHetTable * pHetTable; @@ -1058,33 +1066,37 @@ TMPQHetTable * CreateHetTable(DWORD dwFileCount, DWORD dwNameHashBitSize, LPBYTE pHetTable->AndMask64 = ((dwNameHashBitSize != 0x40) ? ((ULONGLONG)1 << dwNameHashBitSize) : 0) - 1; pHetTable->OrMask64 = (ULONGLONG)1 << (dwNameHashBitSize - 1); + // If the total count is not entered, use default + if(dwTotalCount == 0) + dwTotalCount = (dwEntryCount * 4) / 3; + // Store the HET table parameters - pHetTable->dwEntryCount = dwFileCount; - pHetTable->dwTotalCount = (dwFileCount * 4) / 3; + pHetTable->dwEntryCount = dwEntryCount; + pHetTable->dwTotalCount = dwTotalCount; pHetTable->dwNameHashBitSize = dwNameHashBitSize; - pHetTable->dwIndexSizeTotal = GetNecessaryBitCount(dwFileCount); + pHetTable->dwIndexSizeTotal = GetNecessaryBitCount(dwEntryCount); pHetTable->dwIndexSizeExtra = 0; pHetTable->dwIndexSize = pHetTable->dwIndexSizeTotal; // Allocate array of hashes - pHetTable->pNameHashes = STORM_ALLOC(BYTE, pHetTable->dwTotalCount); + pHetTable->pNameHashes = STORM_ALLOC(BYTE, dwTotalCount); if(pHetTable->pNameHashes != NULL) { // Make sure the data are initialized - memset(pHetTable->pNameHashes, 0, pHetTable->dwTotalCount); + memset(pHetTable->pNameHashes, 0, dwTotalCount); // Allocate the bit array for file indexes - pHetTable->pBetIndexes = CreateBitArray(pHetTable->dwTotalCount * pHetTable->dwIndexSizeTotal, 0xFF); + pHetTable->pBetIndexes = CreateBitArray(dwTotalCount * pHetTable->dwIndexSizeTotal, 0xFF); if(pHetTable->pBetIndexes != NULL) { // Initialize the HET table from the source data (if given) if(pbSrcData != NULL) { // Copy the name hashes - memcpy(pHetTable->pNameHashes, pbSrcData, pHetTable->dwTotalCount); + memcpy(pHetTable->pNameHashes, pbSrcData, dwTotalCount); // Copy the file indexes - memcpy(pHetTable->pBetIndexes->Elements, pbSrcData + pHetTable->dwTotalCount, pHetTable->pBetIndexes->NumberOfBytes); + memcpy(pHetTable->pBetIndexes->Elements, pbSrcData + dwTotalCount, pHetTable->pBetIndexes->NumberOfBytes); } // Return the result HET table @@ -1152,7 +1164,7 @@ static TMPQHetTable * TranslateHetTable(TMPQHetHeader * pHetHeader) assert(pHetHeader->ExtHdr.dwVersion == 1); // Verify size of the HET table - if(pHetHeader->ExtHdr.dwDataSize >= sizeof(TMPQHetHeader)) + if(pHetHeader->ExtHdr.dwDataSize >= (sizeof(TMPQHetHeader) - sizeof(TMPQExtHeader))) { // Verify the size of the table in the header if(pHetHeader->dwTableSize == pHetHeader->ExtHdr.dwDataSize) @@ -1161,7 +1173,8 @@ static TMPQHetTable * TranslateHetTable(TMPQHetHeader * pHetHeader) assert((sizeof(TMPQHetHeader) - sizeof(TMPQExtHeader) + pHetHeader->dwTotalCount + pHetHeader->dwIndexTableSize) == pHetHeader->dwTableSize); // So far, all MPQs with HET Table have had total number of entries equal to 4/3 of file count - assert(((pHetHeader->dwEntryCount * 4) / 3) == pHetHeader->dwTotalCount); + // Exception: "2010 - Starcraft II\!maps\Tya's Zerg Defense (unprotected).SC2Map" +// assert(((pHetHeader->dwEntryCount * 4) / 3) == pHetHeader->dwTotalCount); // The size of one index is predictable as well assert(GetNecessaryBitCount(pHetHeader->dwEntryCount) == pHetHeader->dwIndexSizeTotal); @@ -1171,7 +1184,7 @@ static TMPQHetTable * TranslateHetTable(TMPQHetHeader * pHetHeader) assert(((pHetHeader->dwTotalCount * pHetHeader->dwIndexSizeTotal) + 7) / 8 == pHetHeader->dwIndexTableSize); // Create translated table - pHetTable = CreateHetTable(pHetHeader->dwEntryCount, pHetHeader->dwNameHashBitSize, pbSrcData); + pHetTable = CreateHetTable(pHetHeader->dwEntryCount, pHetHeader->dwTotalCount, pHetHeader->dwNameHashBitSize, pbSrcData); if(pHetTable != NULL) { // Now the sizes in the hash table should be already set @@ -1422,7 +1435,7 @@ static TMPQBetTable * TranslateBetTable( ha = ha; // Verify size of the HET table - if(pBetHeader->ExtHdr.dwDataSize >= sizeof(TMPQBetHeader)) + if(pBetHeader->ExtHdr.dwDataSize >= (sizeof(TMPQBetHeader) - sizeof(TMPQExtHeader))) { // Verify the size of the table in the header if(pBetHeader->dwTableSize == pBetHeader->ExtHdr.dwDataSize) @@ -2263,9 +2276,8 @@ TMPQHetTable * LoadHetTable(TMPQArchive * ha) TMPQHetTable * pHetTable = NULL; TMPQHeader * pHeader = ha->pHeader; - // If the HET table position is not NULL, we expect - // both HET and BET tables to be present. - if(pHeader->HetTablePos64 != 0) + // If the HET table position is not 0, we expect the table to be present + if(pHeader->HetTablePos64 != 0 && pHeader->HetTableSize64 != 0) { // Attempt to load the HET table (Hash Extended Table) pExtTable = LoadExtTable(ha, pHeader->HetTablePos64, (size_t)pHeader->HetTableSize64, HET_TABLE_SIGNATURE, MPQ_KEY_HASH_TABLE); @@ -2286,9 +2298,8 @@ TMPQBetTable * LoadBetTable(TMPQArchive * ha) TMPQBetTable * pBetTable = NULL; TMPQHeader * pHeader = ha->pHeader; - // If the HET table position is not NULL, we expect - // both HET and BET tables to be present. - if(pHeader->BetTablePos64 != 0) + // If the BET table position is not 0, we expect the table to be present + if(pHeader->BetTablePos64 != 0 && pHeader->BetTableSize64 != 0) { // Attempt to load the HET table (Hash Extended Table) pExtTable = LoadExtTable(ha, pHeader->BetTablePos64, (size_t)pHeader->BetTableSize64, BET_TABLE_SIGNATURE, MPQ_KEY_BLOCK_TABLE); @@ -2312,9 +2323,15 @@ int LoadAnyHashTable(TMPQArchive * ha) if(pHeader->dwHashTableSize == 0 && pHeader->HetTableSize64 == 0) return CreateHashTable(ha, HASH_TABLE_SIZE_DEFAULT); - // Try to load HET table (the new hash table) and the classic HASH table - ha->pHetTable = LoadHetTable(ha); - ha->pHashTable = LoadHashTable(ha); + // Try to load HET table + if(pHeader->HetTablePos64 != 0) + ha->pHetTable = LoadHetTable(ha); + + // Try to load the hash table + if(pHeader->wHashTablePosHi || pHeader->dwHashTablePos) + ha->pHashTable = LoadHashTable(ha); + + // At least one of the tables must be present if(ha->pHetTable == NULL && ha->pHashTable == NULL) return ERROR_FILE_CORRUPT; @@ -2580,7 +2597,7 @@ int RebuildHetTable(TMPQArchive * ha) // Create new HET table based on the total number of entries in the file table // Note that if we fail to create it, we just stop using HET table - ha->pHetTable = CreateHetTable(ha->dwFileTableSize, 0x40, NULL); + ha->pHetTable = CreateHetTable(ha->dwFileTableSize, 0, 0x40, NULL); if(ha->pHetTable != NULL) { // Go through the file table again and insert all existing files diff --git a/src/SFileAddFile.cpp b/src/SFileAddFile.cpp index 5246f89..fdc1524 100644 --- a/src/SFileAddFile.cpp +++ b/src/SFileAddFile.cpp @@ -815,7 +815,7 @@ bool WINAPI SFileAddFileEx( // Open added file if(nError == ERROR_SUCCESS) { - pStream = FileStream_OpenFile(szFileName, STREAM_FLAG_READ_ONLY | STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE); + pStream = FileStream_OpenFile(szFileName, STREAM_FLAG_READ_ONLY | STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE); if(pStream == NULL) nError = GetLastError(); } diff --git a/src/SFileAttributes.cpp b/src/SFileAttributes.cpp index a0ba774..7e63993 100644 --- a/src/SFileAttributes.cpp +++ b/src/SFileAttributes.cpp @@ -51,8 +51,7 @@ static DWORD GetSizeOfAttributesFile(DWORD dwAttrFlags, DWORD dwFileTableSize) return cbAttrFile; } -#ifdef _DEBUG -static DWORD GetSizeOfAttributesFile_v2(DWORD dwAttrFlags, DWORD dwFileTableSize) +static bool CheckSizeOfAttributesFile(DWORD cbSizeOfAttr, DWORD dwAttrFlags, DWORD dwFileTableSize) { DWORD cbAttrFile = sizeof(MPQ_ATTRIBUTES_HEADER); @@ -64,15 +63,31 @@ static DWORD GetSizeOfAttributesFile_v2(DWORD dwAttrFlags, DWORD dwFileTableSize if(dwAttrFlags & MPQ_ATTRIBUTE_MD5) cbAttrFile += dwFileTableSize * MD5_DIGEST_SIZE; - // interface.MPQ.part from WoW build 10958 has - // the MPQ_ATTRIBUTE_PATCH_BIT set, but there's an array of DWORDs instead. - // The array is filled with zeros, so we don't know what it should contain + // Various variants with the patch bit if(dwAttrFlags & MPQ_ATTRIBUTE_PATCH_BIT) - cbAttrFile += dwFileTableSize * sizeof(DWORD); - - return cbAttrFile; + { + // Check for expected size + if(cbSizeOfAttr == (cbAttrFile + (dwFileTableSize + 6) / 8)) + return true; + + // Zenith.SC2MAP has the MPQ_ATTRIBUTE_PATCH_BIT set, but the bit array is missing + if(cbSizeOfAttr == cbSizeOfAttr) + return false; + + // interface.MPQ.part from WoW build 10958 has + // the MPQ_ATTRIBUTE_PATCH_BIT set, but there's an array of DWORDs instead. + // The array is filled with zeros, so we don't know what it should contain + if(cbSizeOfAttr == (cbAttrFile + dwFileTableSize * sizeof(DWORD))) + return false; + + // Size mismatch + assert(false); + return false; + } + + assert(cbSizeOfAttr == cbAttrFile); + return false; } -#endif static int LoadAttributesFile(TMPQArchive * ha, LPBYTE pbAttrFile, DWORD cbAttrFile) { @@ -80,6 +95,7 @@ static int LoadAttributesFile(TMPQArchive * ha, LPBYTE pbAttrFile, DWORD cbAttrF LPBYTE pbAttrPtr = pbAttrFile; DWORD dwBlockTableSize = ha->pHeader->dwBlockTableSize; DWORD i; + bool bPatchBitsValid; // Load and verify the header if((pbAttrPtr + sizeof(MPQ_ATTRIBUTES_HEADER)) <= pbAttrFileEnd) @@ -90,15 +106,10 @@ static int LoadAttributesFile(TMPQArchive * ha, LPBYTE pbAttrFile, DWORD cbAttrF if(pAttrHeader->dwVersion != MPQ_ATTRIBUTES_V1) return ERROR_BAD_FORMAT; - // Verify the size of the file + // Verify the flags and of the file assert((pAttrHeader->dwFlags & ~MPQ_ATTRIBUTE_ALL) == 0); - // Verify the size of the file -#ifdef _DEBUG - assert(cbAttrFile == GetSizeOfAttributesFile(pAttrHeader->dwFlags, dwBlockTableSize) || - cbAttrFile == GetSizeOfAttributesFile_v2(pAttrHeader->dwFlags, dwBlockTableSize)); -#endif - + bPatchBitsValid = CheckSizeOfAttributesFile(cbAttrFile, pAttrHeader->dwFlags, dwBlockTableSize); ha->dwAttrFlags = pAttrHeader->dwFlags; pbAttrPtr = (LPBYTE)(pAttrHeader + 1); } @@ -154,7 +165,7 @@ static int LoadAttributesFile(TMPQArchive * ha, LPBYTE pbAttrFile, DWORD cbAttrF } // Read the patch bit for each file (if present) - if(ha->dwAttrFlags & MPQ_ATTRIBUTE_PATCH_BIT) + if((ha->dwAttrFlags & MPQ_ATTRIBUTE_PATCH_BIT) && bPatchBitsValid) { LPBYTE pbBitArray = pbAttrPtr; DWORD cbArraySize = (dwBlockTableSize + 7) / 8; diff --git a/src/SFileCompactArchive.cpp b/src/SFileCompactArchive.cpp index b5482c8..00efbb4 100644 --- a/src/SFileCompactArchive.cpp +++ b/src/SFileCompactArchive.cpp @@ -504,7 +504,7 @@ bool WINAPI SFileCompactArchive(HANDLE hMpq, const char * szListFile, bool /* bR else _tcscat(szTempFile, _T("_")); - pTempStream = FileStream_CreateFile(szTempFile, STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE); + pTempStream = FileStream_CreateFile(szTempFile, STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE); if(pTempStream == NULL) nError = GetLastError(); } diff --git a/src/SFileCreateArchive.cpp b/src/SFileCreateArchive.cpp index 4ae71e9..5f375c8 100644 --- a/src/SFileCreateArchive.cpp +++ b/src/SFileCreateArchive.cpp @@ -77,7 +77,7 @@ bool WINAPI SFileCreateArchive(const TCHAR * szMpqName, DWORD dwCreateFlags, DWO memset(&CreateInfo, 0, sizeof(SFILE_CREATE_MPQ)); CreateInfo.cbSize = sizeof(SFILE_CREATE_MPQ); CreateInfo.dwMpqVersion = (dwCreateFlags & MPQ_CREATE_ARCHIVE_VMASK) >> FLAGS_TO_FORMAT_SHIFT; - CreateInfo.dwStreamFlags = STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE; + CreateInfo.dwStreamFlags = STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE; CreateInfo.dwFileFlags1 = (dwCreateFlags & MPQ_CREATE_LISTFILE) ? MPQ_FILE_EXISTS : 0; CreateInfo.dwFileFlags2 = (dwCreateFlags & MPQ_CREATE_ATTRIBUTES) ? MPQ_FILE_EXISTS : 0; CreateInfo.dwAttrFlags = (dwCreateFlags & MPQ_CREATE_ATTRIBUTES) ? (MPQ_ATTRIBUTE_CRC32 | MPQ_ATTRIBUTE_FILETIME | MPQ_ATTRIBUTE_MD5) : 0; @@ -133,7 +133,7 @@ bool WINAPI SFileCreateArchive2(const TCHAR * szMpqName, PSFILE_CREATE_MPQ pCrea // We verify if the file already exists and if it's a MPQ archive. // If yes, we won't allow to overwrite it. - if(SFileOpenArchive(szMpqName, 0, STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE | MPQ_OPEN_NO_ATTRIBUTES | MPQ_OPEN_NO_LISTFILE, &hMpq)) + if(SFileOpenArchive(szMpqName, 0, STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE | MPQ_OPEN_NO_ATTRIBUTES | MPQ_OPEN_NO_LISTFILE, &hMpq)) { SFileCloseArchive(hMpq); SetLastError(ERROR_ALREADY_EXISTS); @@ -227,7 +227,7 @@ bool WINAPI SFileCreateArchive2(const TCHAR * szMpqName, PSFILE_CREATE_MPQ pCrea // Create initial HET table, if the caller required an MPQ format 3.0 or newer if(nError == ERROR_SUCCESS && pCreateInfo->dwMpqVersion >= MPQ_FORMAT_VERSION_3 && pCreateInfo->dwMaxFileCount != 0) { - ha->pHetTable = CreateHetTable(ha->dwFileTableSize, 0x40, NULL); + ha->pHetTable = CreateHetTable(ha->dwFileTableSize, 0, 0x40, NULL); if(ha->pHetTable == NULL) nError = ERROR_NOT_ENOUGH_MEMORY; } diff --git a/src/SFileExtractFile.cpp b/src/SFileExtractFile.cpp index c8053ed..314d8ff 100644 --- a/src/SFileExtractFile.cpp +++ b/src/SFileExtractFile.cpp @@ -28,7 +28,7 @@ bool WINAPI SFileExtractFile(HANDLE hMpq, const char * szToExtract, const TCHAR // Create the local file if(nError == ERROR_SUCCESS) { - pLocalFile = FileStream_CreateFile(szExtracted, STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE); + pLocalFile = FileStream_CreateFile(szExtracted, 0); if(pLocalFile == NULL) nError = GetLastError(); } diff --git a/src/SFileGetFileInfo.cpp b/src/SFileGetFileInfo.cpp index e56e2f6..05bea29 100644 --- a/src/SFileGetFileInfo.cpp +++ b/src/SFileGetFileInfo.cpp @@ -163,20 +163,10 @@ bool WINAPI SFileGetFileInfo( } break; - case SFileMpqStreamBlockSize: + case SFileMpqStreamBitmap: ha = IsValidMpqHandle(hMpqOrFile); if(ha != NULL) - { - // TODO - } - break; - - case SFileMpqStreamBlockAvailable: - ha = IsValidMpqHandle(hMpqOrFile); - if(ha != NULL) - { - // TODO - } + return FileStream_GetBitmap(ha->pStream, pvFileInfo, cbFileInfo, pcbLengthNeeded); break; case SFileMpqUserDataOffset: @@ -616,7 +606,7 @@ bool WINAPI SFileGetFileInfo( ha = IsValidMpqHandle(hMpqOrFile); if(ha != NULL) { - dwInt32Value = (FileStream_IsReadOnly(ha->pStream) || (ha->dwFlags & MPQ_FLAG_READ_ONLY)); + dwInt32Value = (ha->dwFlags & MPQ_FLAG_READ_ONLY) ? 1 : 0; pvSrcFileInfo = &dwInt32Value; cbSrcFileInfo = sizeof(DWORD); nInfoType = SFILE_INFO_TYPE_DIRECT_POINTER; @@ -954,7 +944,7 @@ static int CreatePseudoFileName(HANDLE hFile, TFileEntry * pFileEntry, char * sz } } - return ERROR_NOT_SUPPORTED; + return ERROR_CAN_NOT_COMPLETE; } bool WINAPI SFileGetFileName(HANDLE hFile, char * szFileName) diff --git a/src/SFileOpenArchive.cpp b/src/SFileOpenArchive.cpp index 9970791..27c9972 100644 --- a/src/SFileOpenArchive.cpp +++ b/src/SFileOpenArchive.cpp @@ -177,7 +177,7 @@ bool WINAPI SFileOpenArchive( { FileStream_GetSize(pStream, &FileSize); if(FileSize < MPQ_HEADER_SIZE_V1) - nError = ERROR_FILE_CORRUPT; + nError = ERROR_BAD_FORMAT; } // Allocate the MPQhandle @@ -191,6 +191,7 @@ bool WINAPI SFileOpenArchive( if(nError == ERROR_SUCCESS) { ULONGLONG SearchOffset = 0; + DWORD dwStreamFlags = 0; DWORD dwHeaderID; memset(ha, 0, sizeof(TMPQArchive)); @@ -198,9 +199,9 @@ bool WINAPI SFileOpenArchive( ha->pStream = pStream; pStream = NULL; - // Remember if the archive is open for write - if(FileStream_IsReadOnly(ha->pStream)) - ha->dwFlags |= MPQ_FLAG_READ_ONLY; + // Set the archive read only if the stream is read-only + FileStream_GetFlags(ha->pStream, &dwStreamFlags); + ha->dwFlags |= (dwStreamFlags & STREAM_FLAG_READ_ONLY) ? MPQ_FLAG_READ_ONLY : 0; // Also remember if we shall check sector CRCs when reading file if(dwFlags & MPQ_OPEN_CHECK_SECTOR_CRC) @@ -287,7 +288,7 @@ bool WINAPI SFileOpenArchive( ha->MpqPos = SearchOffset; // Sector size must be nonzero. - if(ha->pHeader->wSectorSize == 0) + if(SearchOffset >= FileSize || ha->pHeader->wSectorSize == 0) nError = ERROR_BAD_FORMAT; } } @@ -409,6 +410,26 @@ bool WINAPI SFileOpenArchive( } //----------------------------------------------------------------------------- +// bool WINAPI SFileSetDownloadCallback(HANDLE, SFILE_DOWNLOAD_CALLBACK, void *); +// +// Sets a callback that is called when content is downloaded from the master MPQ +// + +bool WINAPI SFileSetDownloadCallback(HANDLE hMpq, SFILE_DOWNLOAD_CALLBACK DownloadCB, void * pvUserData) +{ + TMPQArchive * ha = (TMPQArchive *)hMpq; + + // Do nothing if 'hMpq' is bad parameter + if(!IsValidMpqHandle(hMpq)) + { + SetLastError(ERROR_INVALID_HANDLE); + return false; + } + + return FileStream_SetCallback(ha->pStream, DownloadCB, pvUserData); +} + +//----------------------------------------------------------------------------- // bool SFileFlushArchive(HANDLE hMpq) // // Saves all dirty data into MPQ archive. diff --git a/src/SFileOpenFileEx.cpp b/src/SFileOpenFileEx.cpp index c2cf5d4..31e235e 100644 --- a/src/SFileOpenFileEx.cpp +++ b/src/SFileOpenFileEx.cpp @@ -46,7 +46,7 @@ static bool OpenLocalFile(const char * szFileName, HANDLE * phFile) CopyFileName(szFileNameT, szFileName, strlen(szFileName)); // Open the file and create the TMPQFile structure - pStream = FileStream_OpenFile(szFileNameT, STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE | STREAM_FLAG_READ_ONLY); + pStream = FileStream_OpenFile(szFileNameT, STREAM_FLAG_READ_ONLY); if(pStream != NULL) { // Allocate and initialize file handle diff --git a/src/SFilePatchArchives.cpp b/src/SFilePatchArchives.cpp index cc10726..4b49c59 100644 --- a/src/SFilePatchArchives.cpp +++ b/src/SFilePatchArchives.cpp @@ -519,7 +519,7 @@ bool WINAPI SFileOpenPatchArchive( if(nError == ERROR_SUCCESS) { - if(!FileStream_IsReadOnly(ha->pStream)) + if(!(ha->dwFlags & MPQ_FLAG_READ_ONLY)) nError = ERROR_ACCESS_DENIED; } diff --git a/src/StormCommon.h b/src/StormCommon.h index 2901c9f..2b2cf1d 100644 --- a/src/StormCommon.h +++ b/src/StormCommon.h @@ -193,7 +193,7 @@ int RebuildHetTable(TMPQArchive * ha); int RebuildFileTable(TMPQArchive * ha, DWORD dwNewHashTableSize, DWORD dwNewMaxFileCount); int SaveMPQTables(TMPQArchive * ha); -TMPQHetTable * CreateHetTable(DWORD dwFileCount, DWORD dwHashBitSize, LPBYTE pbSrcData); +TMPQHetTable * CreateHetTable(DWORD dwEntryCount, DWORD dwTotalCount, DWORD dwHashBitSize, LPBYTE pbSrcData); void FreeHetTable(TMPQHetTable * pHetTable); TMPQBetTable * CreateBetTable(DWORD dwMaxFileCount); diff --git a/src/StormLib.h b/src/StormLib.h index e528283..a0cfeb6 100644 --- a/src/StormLib.h +++ b/src/StormLib.h @@ -145,6 +145,7 @@ extern "C" { #define ERROR_INTERNAL_FILE 10003 // The given operation is not allowed on internal file #define ERROR_BASE_FILE_MISSING 10004 // The file is present as incremental patch file, but base file is missing #define ERROR_MARKED_FOR_DELETE 10005 // The file was marked as "deleted" in the MPQ +#define ERROR_FILE_INCOMPLETE 10006 // The required file part is missing // Values for SFileCreateArchive #define HASH_TABLE_SIZE_MIN 0x00000004 // Verified: If there is 1 file, hash table size is 4 @@ -264,9 +265,10 @@ extern "C" { #define BASE_PROVIDER_HTTP 0x00000002 // Base data source is a file on web server #define BASE_PROVIDER_MASK 0x0000000F // Mask for base provider value -#define STREAM_PROVIDER_LINEAR 0x00000000 // Stream is linear with no offset mapping +#define STREAM_PROVIDER_FLAT 0x00000000 // Stream is linear with no offset mapping #define STREAM_PROVIDER_PARTIAL 0x00000010 // Stream is partial file (.part) -#define STREAM_PROVIDER_ENCRYPTED 0x00000020 // Stream is an encrypted MPQ +#define STREAM_PROVIDER_MPQE 0x00000020 // Stream is an encrypted MPQ +#define STREAM_PROVIDER_BLOCK4 0x00000030 // 0x4000 per block, text MD5 after each block, max 0x2000 blocks per file #define STREAM_PROVIDER_MASK 0x000000F0 // Mask for stream provider value #define STREAM_FLAG_READ_ONLY 0x00000100 // Stream is read only @@ -282,11 +284,7 @@ extern "C" { #define MPQ_OPEN_NO_HEADER_SEARCH 0x00040000 // Don't search for the MPQ header past the begin of the file #define MPQ_OPEN_FORCE_MPQ_V1 0x00080000 // Always open the archive as MPQ v 1.00, ignore the "wFormatVersion" variable in the header #define MPQ_OPEN_CHECK_SECTOR_CRC 0x00100000 // On files with MPQ_FILE_SECTOR_CRC, the CRC will be checked when reading file - -// Deprecated #define MPQ_OPEN_READ_ONLY STREAM_FLAG_READ_ONLY -#define MPQ_OPEN_ENCRYPTED STREAM_PROVIDER_ENCRYPTED -#define MPQ_OPEN_PARTIAL STREAM_PROVIDER_PARTIAL // Flags for SFileCreateArchive #define MPQ_CREATE_LISTFILE 0x00100000 // Also add the (listfile) file @@ -363,8 +361,7 @@ typedef enum _SFileInfoClass { // Info classes for archives SFileMpqFileName, // Name of the archive file (TCHAR []) - SFileMpqStreamBlockSize, // Size of one stream block in bytes (DWORD) - SFileMpqStreamBlockAvailable, // Nonzero if the stream block at the given offset is available (input: ByteOffset, ULONGLONG) + SFileMpqStreamBitmap, // Array of bits, each bit means availability of one block (BYTE []) SFileMpqUserDataOffset, // Offset of the user data header (ULONGLONG) SFileMpqUserDataHeader, // Raw (unfixed) user data header (TMPQUserData) SFileMpqUserData, // MPQ USer data, without the header (BYTE []) @@ -426,6 +423,11 @@ typedef enum _SFileInfoClass //----------------------------------------------------------------------------- // Deprecated flags. These are going to be removed in next releases. +STORMLIB_DEPRECATED_FLAG(DWORD, STREAM_PROVIDER_LINEAR, STREAM_PROVIDER_FLAT); +STORMLIB_DEPRECATED_FLAG(DWORD, STREAM_PROVIDER_ENCRYPTED, STREAM_PROVIDER_MPQE); +STORMLIB_DEPRECATED_FLAG(DWORD, MPQ_OPEN_ENCRYPTED, STREAM_PROVIDER_MPQE); +STORMLIB_DEPRECATED_FLAG(DWORD, MPQ_OPEN_PARTIAL, STREAM_PROVIDER_PARTIAL); + // MPQ_FILE_COMPRESSED is deprecated. Do not use. STORMLIB_DEPRECATED_FLAG(DWORD, MPQ_FILE_COMPRESSED, MPQ_FILE_COMPRESS_MASK); @@ -465,6 +467,7 @@ STORMLIB_DEPRECATED_FLAG(SFileInfoClass, SFILE_INFO_PATCH_CHAIN, SFileInfoPatchC #define CCB_COMPACTING_FILES 4 // Compacting archive (dwParam1 = current, dwParam2 = total) #define CCB_CLOSING_ARCHIVE 5 // Closing archive: No params used +typedef void (WINAPI * SFILE_DOWNLOAD_CALLBACK)(void * pvUserData, ULONGLONG ByteOffset, DWORD dwTotalBytes); typedef void (WINAPI * SFILE_ADDFILE_CALLBACK)(void * pvUserData, DWORD dwBytesWritten, DWORD dwTotalBytes, bool bFinalCall); typedef void (WINAPI * SFILE_COMPACT_CALLBACK)(void * pvUserData, DWORD dwWorkType, ULONGLONG BytesProcessed, ULONGLONG TotalBytes); @@ -931,12 +934,28 @@ typedef struct _SFILE_CREATE_MPQ //----------------------------------------------------------------------------- // Stream support - functions +// Structure used by FileStream_GetBitmap +typedef struct _TStreamBitmap +{ + ULONGLONG StreamSize; // Size of the stream, in bytes + DWORD BitmapSize; // Size of the block map, in bytes + DWORD BlockCount; // Number of blocks in the stream + DWORD BlockSize; // Size of one block + DWORD IsComplete; // Nonzero if the file is complete + + // Followed by the BYTE array, each bit means availability of one block + +} TStreamBitmap; + // UNICODE versions of the file access functions TFileStream * FileStream_CreateFile(const TCHAR * szFileName, DWORD dwStreamFlags); TFileStream * FileStream_OpenFile(const TCHAR * szFileName, DWORD dwStreamFlags); const TCHAR * FileStream_GetFileName(TFileStream * pStream); +size_t FileStream_Prefix(const TCHAR * szFileName, DWORD * pdwProvider); + +bool FileStream_SetCallback(TFileStream * pStream, SFILE_DOWNLOAD_CALLBACK pfnCallback, void * pvUserData); -bool FileStream_IsReadOnly(TFileStream * pStream); +bool FileStream_GetBitmap(TFileStream * pStream, void * pvBitmap, DWORD cbBitmap, LPDWORD pcbLengthNeeded); bool FileStream_Read(TFileStream * pStream, ULONGLONG * pByteOffset, void * pvBuffer, DWORD dwBytesToRead); bool FileStream_Write(TFileStream * pStream, ULONGLONG * pByteOffset, const void * pvBuffer, DWORD dwBytesToWrite); bool FileStream_SetSize(TFileStream * pStream, ULONGLONG NewFileSize); @@ -973,6 +992,7 @@ bool WINAPI SFileOpenArchive(const TCHAR * szMpqName, DWORD dwPriority, DWORD bool WINAPI SFileCreateArchive(const TCHAR * szMpqName, DWORD dwCreateFlags, DWORD dwMaxFileCount, HANDLE * phMpq); bool WINAPI SFileCreateArchive2(const TCHAR * szMpqName, PSFILE_CREATE_MPQ pCreateInfo, HANDLE * phMpq); +bool WINAPI SFileSetDownloadCallback(HANDLE hMpq, SFILE_DOWNLOAD_CALLBACK DownloadCB, void * pvUserData); bool WINAPI SFileFlushArchive(HANDLE hMpq); bool WINAPI SFileCloseArchive(HANDLE hMpq); diff --git a/test/TLogHelper.cpp b/test/TLogHelper.cpp index 6bad08e..1e0c787 100644 --- a/test/TLogHelper.cpp +++ b/test/TLogHelper.cpp @@ -60,9 +60,13 @@ class TLogHelper #ifdef _MSC_VER #define I64u_t _T("%I64u") #define I64u_a "%I64u" +#define I64X_t _T("%I64X") +#define I64X_a "%I64X" #else -#define I64u_t "%llu" +#define I64u_t _T("%llu") #define I64u_a "%llu" +#define I64X_t _T("%llX") +#define I64X_a "%llX" #endif //----------------------------------------------------------------------------- diff --git a/test/Test.cpp b/test/Test.cpp index 30b2fd4..6f27f1e 100644 --- a/test/Test.cpp +++ b/test/Test.cpp @@ -174,6 +174,19 @@ static const char * PatchList_WoW16965[] = static char szMpqDirectory[MAX_PATH]; size_t cchMpqDirectory = 0; +static bool IsFullPath(const char * szFileName) +{ +#ifdef PLATFORM_WINDOWS + if(('A' <= szFileName[0] && szFileName[0] <= 'Z') || ('a' <= szFileName[0] && szFileName[0] <= 'z')) + { + return (szFileName[1] == ':' && szFileName[2] == PATH_SEPARATOR); + } +#endif + + szFileName = szFileName; + return false; +} + static bool IsMpqExtension(const char * szFileName) { const char * szExtension = strrchr(szFileName, '.'); @@ -196,6 +209,8 @@ static bool IsMpqExtension(const char * szFileName) return true; if(!_stricmp(szExtension, ".SC2Map")) return true; + if(!_stricmp(szExtension, ".0")) // .MPQ.0 + return true; // if(!_stricmp(szExtension, ".link")) // return true; } @@ -290,6 +305,19 @@ static void CopyPathPart(char * szBuffer, const char * szPath) *szBuffer = 0; } +static bool CopyStringAndVerifyConversion( + const TCHAR * szFoundFile, + TCHAR * szBufferT, + char * szBufferA) +{ + // Convert the TCHAR name to ANSI name + CopyFileName(szBufferA, szFoundFile, _tcslen(szFoundFile)); + CopyFileName(szBufferT, szBufferA, strlen(szBufferA)); + + // Compare both TCHAR strings + return (_tcsicmp(szBufferT, szFoundFile) == 0) ? true : false; +} + static void CalculateRelativePath(const char * szFullPath1, const char * szFullPath2, char * szBuffer) { const char * szPathPart1 = szFullPath1; @@ -346,44 +374,98 @@ static void CalculateRelativePath(const char * szFullPath1, const char * szFullP strcpy(szBuffer, szFullPath1); } +static TFileStream * FileStream_OpenFileA(const char * szFileName, DWORD dwStreamFlags) +{ + TCHAR szFileNameT[MAX_PATH]; + + CopyFileName(szFileNameT, szFileName, strlen(szFileName)); + return FileStream_OpenFile(szFileNameT, dwStreamFlags); +} + +static TFileStream * FileStream_CreateFileA(const char * szFileName, DWORD dwStreamFlags) +{ + TCHAR szFileNameT[MAX_PATH]; + + CopyFileName(szFileNameT, szFileName, strlen(szFileName)); + return FileStream_CreateFile(szFileNameT, dwStreamFlags); +} + +static size_t FileStream_PrefixA(const char * szFileName, DWORD * pdwProvider) +{ + TCHAR szFileNameT[MAX_PATH]; + + if(szFileName != NULL) + { + CopyFileName(szFileNameT, szFileName, strlen(szFileName)); + return FileStream_Prefix(szFileNameT, pdwProvider); + } + return 0; +} + static void CreateFullPathName(char * szBuffer, const char * szSubDir, const char * szNamePart1, const char * szNamePart2 = NULL) { + char * szSaveBuffer = szBuffer; + size_t nPrefixLength = 0; size_t nLength; + DWORD dwProvider = 0; + bool bIsFullPath = false; + char chSeparator = PATH_SEPARATOR; - // Copy the master MPQ directory - memcpy(szBuffer, szMpqDirectory, cchMpqDirectory); - szBuffer += cchMpqDirectory; + // Determine the path prefix + if(szNamePart1 != NULL) + { + nPrefixLength = FileStream_PrefixA(szNamePart1, &dwProvider); + if((dwProvider & BASE_PROVIDER_MASK) == BASE_PROVIDER_HTTP) + { + bIsFullPath = true; + chSeparator = '/'; + } + else + bIsFullPath = IsFullPath(szNamePart1 + nPrefixLength); + } - // Append the subdirectory, if any - if(szSubDir != NULL && (nLength = strlen(szSubDir)) != 0) + // Copy the MPQ prefix, if any + if(nPrefixLength > 0) { - // No leading or trailing separator must be there - assert(szSubDir[0] != '/' && szSubDir[0] != '\\'); - assert(szSubDir[nLength - 1] != '/' && szSubDir[nLength - 1] != '\\'); + memcpy(szBuffer, szNamePart1, nPrefixLength); + szSaveBuffer += nPrefixLength; + szNamePart1 += nPrefixLength; + szBuffer += nPrefixLength; + } - // Append file path separator - *szBuffer++ = PATH_SEPARATOR; + // If the given name is not a full path, copy the MPQ directory + if(bIsFullPath == false) + { + // Copy the master MPQ directory + memcpy(szBuffer, szMpqDirectory, cchMpqDirectory); + szBuffer += cchMpqDirectory; - // Copy the subdirectory - memcpy(szBuffer, szSubDir, nLength); + // Append the subdirectory, if any + if(szSubDir != NULL && (nLength = strlen(szSubDir)) != 0) + { + // No leading or trailing separator are allowed + assert(szSubDir[0] != '/' && szSubDir[0] != '\\'); + assert(szSubDir[nLength - 1] != '/' && szSubDir[nLength - 1] != '\\'); - // Fix the path separators - for(size_t i = 0; i < nLength; i++) - szBuffer[i] = (szBuffer[i] != '\\' && szBuffer[i] != '/') ? szBuffer[i] : PATH_SEPARATOR; - - // Move the buffer pointer - szBuffer += nLength; + // Append file path separator + *szBuffer++ = PATH_SEPARATOR; + + // Append the subdirectory + memcpy(szBuffer, szSubDir, nLength); + szBuffer += nLength; + } } // Copy the file name, if any if(szNamePart1 != NULL && (nLength = strlen(szNamePart1)) != 0) { - // No path separator can be there - assert(strchr(szNamePart1, '\\') == NULL); - assert(strchr(szNamePart1, '/') == NULL); + // Path separators are not allowed in the name part + assert(szNamePart1[0] != '\\' && szNamePart1[0] != '/'); + assert(szNamePart1[nLength - 1] != '/' && szNamePart1[nLength - 1] != '\\'); // Append file path separator - *szBuffer++ = PATH_SEPARATOR; + if(bIsFullPath == false) + *szBuffer++ = PATH_SEPARATOR; // Copy the file name memcpy(szBuffer, szNamePart1, nLength); @@ -398,47 +480,61 @@ static void CreateFullPathName(char * szBuffer, const char * szSubDir, const cha szBuffer += nLength; } + // Normalize the path separators + while(szSaveBuffer < szBuffer) + { + szSaveBuffer[0] = (szSaveBuffer[0] != '/' && szSaveBuffer[0] != '\\') ? szSaveBuffer[0] : chSeparator; + szSaveBuffer++; + } + // Terminate the buffer with zero *szBuffer = 0; } -static int FindFilesInternal(FIND_FILE_CALLBACK pfnTest, char * szDirectory) +static int FindFilesInternal(FIND_FILE_CALLBACK pfnTest, TCHAR * szDirectoryT, char * szDirectoryA) { - char * szPlainName; int nError = ERROR_SUCCESS; - // Setup the search masks - strcat(szDirectory, "\\*"); - szPlainName = strrchr(szDirectory, '*'); - - if(szDirectory != NULL && szPlainName != NULL) + if(szDirectoryT != NULL && szDirectoryA != NULL) { #ifdef PLATFORM_WINDOWS - WIN32_FIND_DATAA wf; + WIN32_FIND_DATA wf; HANDLE hFind; + TCHAR * szPlainNameT; + char * szPlainNameA; + size_t nLength = strlen(szDirectoryA); + + // Setup the search masks + _tcscat(szDirectoryT, _T("\\*")); + szPlainNameT = szDirectoryT + nLength + 1; + + strcat(szDirectoryA, "\\*"); + szPlainNameA = szDirectoryA + nLength + 1; // Initiate search. Use ANSI function only - hFind = FindFirstFileA(szDirectory, &wf); + hFind = FindFirstFile(szDirectoryT, &wf); if(hFind != INVALID_HANDLE_VALUE) { // Skip the first entry, since it's always "." or ".." - while(FindNextFileA(hFind, &wf) && nError == ERROR_SUCCESS) + while(FindNextFile(hFind, &wf) && nError == ERROR_SUCCESS) { - // Found a directory? - if(wf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + // If the file name can be converted to UNICODE and back + if(CopyStringAndVerifyConversion(wf.cFileName, szPlainNameT, szPlainNameA)) { - if(wf.cFileName[0] != '.') + // Found a directory? + if(wf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - strcpy(szPlainName, wf.cFileName); - nError = FindFilesInternal(pfnTest, szDirectory); + if(wf.cFileName[0] != '.') + { + nError = FindFilesInternal(pfnTest, szDirectoryT, szDirectoryA); + } } - } - else - { - if(pfnTest != NULL) + else { - strcpy(szPlainName, wf.cFileName); - nError = pfnTest(szDirectory); + if(pfnTest != NULL) + { + nError = pfnTest(szDirectoryA); + } } } } @@ -454,10 +550,12 @@ static int FindFilesInternal(FIND_FILE_CALLBACK pfnTest, char * szDirectory) static int FindFiles(FIND_FILE_CALLBACK pfnFindFile, const char * szSubDirectory) { - char szWorkBuff[MAX_PATH]; + TCHAR szWorkBuffT[MAX_PATH]; + char szWorkBuffA[MAX_PATH]; - CreateFullPathName(szWorkBuff, szSubDirectory, NULL); - return FindFilesInternal(pfnFindFile, szWorkBuff); + CreateFullPathName(szWorkBuffA, szSubDirectory, NULL); + CopyFileName(szWorkBuffT, szWorkBuffA, strlen(szWorkBuffA)); + return FindFilesInternal(pfnFindFile, szWorkBuffT, szWorkBuffA); } static int FindFilePairsInternal( @@ -529,22 +627,6 @@ static int FindFilePairs(FIND_PAIR_CALLBACK pfnFindPair, const char * szSourceSu return FindFilePairsInternal(pfnFindPair, szSource, szTarget); } -TFileStream * OpenLocalFile(const char * szFileName, DWORD dwStreamFlags) -{ - TCHAR szFileNameT[MAX_PATH]; - - CopyFileName(szFileNameT, szFileName, strlen(szFileName)); - return FileStream_OpenFile(szFileNameT, dwStreamFlags); -} - -TFileStream * CreateLocalFile(const char * szFileName, DWORD dwStreamFlags) -{ - TCHAR szFileNameT[MAX_PATH]; - - CopyFileName(szFileNameT, szFileName, strlen(szFileName)); - return FileStream_CreateFile(szFileNameT, dwStreamFlags); -} - static int CalculateFileSha1(TLogHelper * pLogger, const char * szFullPath, char * szFileSha1) { TFileStream * pStream; @@ -563,7 +645,7 @@ static int CalculateFileSha1(TLogHelper * pLogger, const char * szFullPath, char szFileSha1[0] = 0; // Open the file to be verified - pStream = OpenLocalFile(szFullPath, STREAM_FLAG_READ_ONLY); + pStream = FileStream_OpenFileA(szFullPath, STREAM_FLAG_READ_ONLY); if(pStream != NULL) { // Retrieve the size of the file @@ -626,7 +708,7 @@ static int InitializeMpqDirectory(char * argv[], int argc) // Retrieve the name of the MPQ directory if(argc > 1 && argv[1] != NULL) { - szWhereFrom = "entered at command line"; + szWhereFrom = "command line"; szDirName = argv[1]; } else @@ -649,16 +731,17 @@ static int InitializeMpqDirectory(char * argv[], int argc) // Verify if the work MPQ directory is writable CreateFullPathName(szFullPath, NULL, "TestFile.bin"); - pStream = CreateLocalFile(szFullPath, 0); + pStream = FileStream_CreateFileA(szFullPath, 0); if(pStream == NULL) - return Logger.PrintError("MPQ subdirectory is not writable"); + return Logger.PrintError("MPQ subdirectory doesn't exist or is not writable"); // Close the stream FileStream_Close(pStream); + remove(szFullPath); // Verify if the working directory exists and if there is a subdirectory with the file name CreateFullPathName(szFullPath, szMpqSubDir, "ListFile_Blizzard.txt"); - pStream = OpenLocalFile(szFullPath, STREAM_FLAG_READ_ONLY); + pStream = FileStream_OpenFileA(szFullPath, STREAM_FLAG_READ_ONLY); if(pStream == NULL) return Logger.PrintError("The main listfile (%s) was not found. Check your paths", GetShortPlainName(szFullPath)); @@ -756,7 +839,7 @@ static int CreateEmptyFile(TLogHelper * pLogger, const char * szPlainName, ULONG // Construct the full path and crete the file CreateFullPathName(szFullPath, NULL, szPlainName); - pStream = CreateLocalFile(szFullPath, STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE); + pStream = FileStream_CreateFileA(szFullPath, STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE); if(pStream == NULL) return pLogger->PrintError("Failed to create file %s", szBuffer); @@ -962,7 +1045,7 @@ static int CreateFileCopy( } // Open the source file - pStream1 = OpenLocalFile(szFileName1, STREAM_FLAG_READ_ONLY); + pStream1 = FileStream_OpenFileA(szFileName1, STREAM_FLAG_READ_ONLY); if(pStream1 == NULL) { pLogger->PrintError("Failed to open the source file %s", szFileName1); @@ -970,7 +1053,7 @@ static int CreateFileCopy( } // Create the destination file - pStream2 = CreateLocalFile(szFileName2, 0); + pStream2 = FileStream_CreateFileA(szFileName2, 0); if(pStream2 != NULL) { // If we should write some pre-MPQ data to the target file, do it @@ -1015,13 +1098,21 @@ static int CreateMasterAndMirrorPaths( char * szMirrorPath, char * szMasterPath, const char * szMirrorName, - const char * szMasterName) + const char * szMasterName, + bool bCopyMirrorFile) { char szCopyPath[MAX_PATH]; - int nError; + int nError = ERROR_SUCCESS; // Copy the mirrored file from the source to the work directory - nError = CreateFileCopy(pLogger, szMirrorName, szMirrorName, szCopyPath); + if(bCopyMirrorFile) + nError = CreateFileCopy(pLogger, szMirrorName, szMirrorName, szCopyPath); + else + { + CreateFullPathName(szCopyPath, NULL, szMirrorName); + remove(szCopyPath); + } + if(nError == ERROR_SUCCESS) { // Create the full path name of the master file @@ -1133,7 +1224,7 @@ static TFileData * LoadLocalFile(TLogHelper * pLogger, const char * szFileName, pLogger->PrintProgress("Loading local file ..."); // Attempt to open the file - pStream = OpenLocalFile(szFileName, STREAM_FLAG_READ_ONLY); + pStream = FileStream_OpenFileA(szFileName, STREAM_FLAG_READ_ONLY); if(pStream == NULL) { if(pLogger != NULL && bMustSucceed == true) @@ -1170,7 +1261,8 @@ static TFileData * LoadLocalFile(TLogHelper * pLogger, const char * szFileName, static int CompareTwoLocalFilesRR( TLogHelper * pLogger, TFileStream * pStream1, // Master file - TFileStream * pStream2) // Mirror file + TFileStream * pStream2, // Mirror file + int nIterations) // Number of iterations { ULONGLONG RandomNumber = 0x12345678; // We need pseudo-random number that will repeat each run of the program ULONGLONG RandomSeed; @@ -1199,7 +1291,7 @@ static int CompareTwoLocalFilesRR( if(pbBuffer1 && pbBuffer2) { // Perform many random reads - for(int i = 0; i < 0x10000; i++) + for(int i = 0; i < nIterations; i++) { // Generate psudo-random offsrt and data size ByteOffset = (RandomNumber % FileSize1); @@ -1495,18 +1587,25 @@ static int OpenExistingArchive(TLogHelper * pLogger, const char * szFullPath, DW // Is it an encrypted MPQ ? if(strstr(szFullPath, ".MPQE") != NULL) - dwFlags |= STREAM_PROVIDER_ENCRYPTED; + dwFlags |= STREAM_PROVIDER_MPQE; if(strstr(szFullPath, ".MPQ.part") != NULL) dwFlags |= STREAM_PROVIDER_PARTIAL; + if(strstr(szFullPath, ".mpq.part") != NULL) + dwFlags |= STREAM_PROVIDER_PARTIAL; + if(strstr(szFullPath, ".MPQ.0") != NULL) + dwFlags |= STREAM_PROVIDER_BLOCK4; // Open the copied archive pLogger->PrintProgress("Opening archive %s ...", GetShortPlainName(szFullPath)); CopyFileName(szMpqName, szFullPath, strlen(szFullPath)); if(!SFileOpenArchive(szMpqName, 0, dwFlags, &hMpq)) { - // Ignore the error if it's an AVI file - if(GetLastError() == ERROR_AVI_FILE) - return ERROR_AVI_FILE; + // Ignore the error if it's an AVI file or if the file is incomplete + nError = GetLastError(); + if(nError == ERROR_AVI_FILE || nError == ERROR_FILE_INCOMPLETE) + return nError; + + // Show the open error to the user return pLogger->PrintError("Failed to open archive %s", szFullPath); } @@ -1810,25 +1909,55 @@ static int TestSearchListFile(const char * szPlainName) return ERROR_SUCCESS; } +static void WINAPI TestReadFile_DownloadCallback( + void * UserData, + ULONGLONG ByteOffset, + DWORD DataLength) +{ + TLogHelper * pLogger = (TLogHelper *)UserData; + + if(ByteOffset != 0 && DataLength != 0) + pLogger->PrintProgress("Downloading data (offset: " I64X_a ", length: %X)", ByteOffset, DataLength); + else + pLogger->PrintProgress("Download complete."); +} + // Open a file stream with mirroring a master file -static int TestReadFile_MasterMirror(const char * szMirrorName, const char * szMasterName) +static int TestReadFile_MasterMirror(const char * szMirrorName, const char * szMasterName, bool bCopyMirrorFile) { TFileStream * pStream1; // Master file TFileStream * pStream2; // Mirror file - TLogHelper Logger("OpenMirrorFile", szMasterName); + TLogHelper Logger("OpenMirrorFile", szMirrorName); + DWORD dwProvider = 0; char szMirrorPath[MAX_PATH + MAX_PATH]; char szMasterPath[MAX_PATH]; + int nIterations = 0x10000; int nError; + // Retrieve the provider + FileStream_PrefixA(szMasterName, &dwProvider); + +#ifndef PLATFORM_WINDOWS + if((dwProvider & BASE_PROVIDER_MASK) == BASE_PROVIDER_HTTP) + return ERROR_SUCCESS; +#endif + // Create copy of the file to serve as mirror, keep master there - nError = CreateMasterAndMirrorPaths(&Logger, szMirrorPath, szMasterPath, szMirrorName, szMasterName); + nError = CreateMasterAndMirrorPaths(&Logger, szMirrorPath, szMasterPath, szMirrorName, szMasterName, bCopyMirrorFile); if(nError == ERROR_SUCCESS) { // Open both master and mirror file - pStream1 = OpenLocalFile(szMasterPath, STREAM_FLAG_READ_ONLY | STREAM_FLAG_USE_BITMAP); - pStream2 = OpenLocalFile(szMirrorPath, STREAM_FLAG_READ_ONLY | STREAM_FLAG_USE_BITMAP); + pStream1 = FileStream_OpenFileA(szMasterPath, STREAM_FLAG_READ_ONLY); + pStream2 = FileStream_OpenFileA(szMirrorPath, STREAM_FLAG_READ_ONLY | STREAM_FLAG_USE_BITMAP); if(pStream1 && pStream2) - nError = CompareTwoLocalFilesRR(&Logger, pStream1, pStream2); + { + // For internet based files, we limit the number of operations + if((dwProvider & BASE_PROVIDER_MASK) == BASE_PROVIDER_HTTP) + nIterations = 0x80; + + FileStream_SetCallback(pStream2, TestReadFile_DownloadCallback, &Logger); + nError = CompareTwoLocalFilesRR(&Logger, pStream1, pStream2, nIterations); + } if(pStream2 != NULL) FileStream_Close(pStream2); @@ -1852,12 +1981,18 @@ static int TestFileStreamOperations(const char * szPlainName, DWORD dwStreamFlag int nError = ERROR_SUCCESS; // Copy the file so we won't screw up - CreateFileCopy(&Logger, szPlainName, szPlainName, szFullPath); + if((dwStreamFlags & STREAM_PROVIDER_MASK) == STREAM_PROVIDER_BLOCK4) + CreateFullPathName(szFullPath, szMpqSubDir, szPlainName); + else + nError = CreateFileCopy(&Logger, szPlainName, szPlainName, szFullPath); // Open the file stream - pStream = OpenLocalFile(szFullPath, dwStreamFlags); - if(pStream == NULL) - nError = Logger.PrintError("Failed to open %s", szFullPath); + if(nError == ERROR_SUCCESS) + { + pStream = FileStream_OpenFileA(szFullPath, dwStreamFlags); + if(pStream == NULL) + nError = Logger.PrintError("Failed to open %s", szFullPath); + } // Get the size of the file stream if(nError == ERROR_SUCCESS) @@ -1868,8 +2003,8 @@ static int TestFileStreamOperations(const char * szPlainName, DWORD dwStreamFlag if(!FileStream_GetSize(pStream, &FileSize)) nError = Logger.PrintError("Failed to retrieve the file size"); - // Any other stream except STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE should be read-only - if((dwStreamFlags & STREAM_PROVIDERS_MASK) != (STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE)) + // Any other stream except STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE should be read-only + if((dwStreamFlags & STREAM_PROVIDERS_MASK) != (STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE)) dwRequiredFlags |= STREAM_FLAG_READ_ONLY; // if(pStream->BlockPresent) // dwRequiredFlags |= STREAM_FLAG_READ_ONLY; @@ -2233,7 +2368,7 @@ static int TestOpenArchive_GetFileInfo(const char * szPlainName1, const char * s return ERROR_SUCCESS; } -static int TestOpenArchive_MasterMirror(const char * szMirrorName, const char * szMasterName, const char * szFileToExtract) +static int TestOpenArchive_MasterMirror(const char * szMirrorName, const char * szMasterName, const char * szFileToExtract, bool bCopyMirrorFile) { TFileData * pFileData; TLogHelper Logger("OpenServerMirror", szMirrorName); @@ -2245,7 +2380,7 @@ static int TestOpenArchive_MasterMirror(const char * szMirrorName, const char * int nError; // Create both paths - nError = CreateMasterAndMirrorPaths(&Logger, szMirrorPath, szMasterPath, szMirrorName, szMasterName); + nError = CreateMasterAndMirrorPaths(&Logger, szMirrorPath, szMasterPath, szMirrorName, szMasterName, bCopyMirrorFile); // Now open both archives as local-server pair if(nError == ERROR_SUCCESS) @@ -2469,7 +2604,7 @@ static int ForEachFile_OpenArchive(const char * szFullPath) // Open the MPQ name nError = OpenExistingArchive(&Logger, szFullPath, 0, &hMpq); - if(nError == ERROR_AVI_FILE) + if(nError == ERROR_AVI_FILE || nError == ERROR_FILE_CORRUPT || nError == ERROR_BAD_FORMAT) return ERROR_SUCCESS; // Search the archive and load every file @@ -2480,6 +2615,9 @@ static int ForEachFile_OpenArchive(const char * szFullPath) } } + // Correct some errors + if(nError == ERROR_FILE_CORRUPT || nError == ERROR_FILE_INCOMPLETE) + return ERROR_SUCCESS; return nError; } @@ -3151,7 +3289,7 @@ static int CreateArchiveLinkFile(const char * szFullPath1, const char * szFullPa nLength = sprintf(szLinkData, "LINK:%s\x0D\x0ASHA1:%s", szLinkPath, szFileHash); // Create the link file - pStream = CreateLocalFile(szLinkFile, 0); + pStream = FileStream_CreateFileA(szLinkFile, 0); if(pStream == NULL) return GetLastError(); @@ -3226,19 +3364,35 @@ int main(int argc, char * argv[]) // Test reading linear file with bitmap if(nError == ERROR_SUCCESS) - nError = TestFileStreamOperations("MPQ_2013_v4_alternate-downloaded.MPQ", STREAM_FLAG_USE_BITMAP); + nError = TestFileStreamOperations("MPQ_2013_v4_alternate-complete.MPQ", STREAM_FLAG_USE_BITMAP); // Test reading partial file if(nError == ERROR_SUCCESS) - nError = TestFileStreamOperations("MPQ_2009_v2_WoW_patch.MPQ.part", STREAM_PROVIDER_PARTIAL); + nError = TestFileStreamOperations("part-file://MPQ_2009_v2_WoW_patch.MPQ.part", 0); + + // Test reading Block4K file + if(nError == ERROR_SUCCESS) + nError = TestFileStreamOperations("blk4-file://streaming/model.MPQ.0", STREAM_PROVIDER_BLOCK4); // Test reading encrypted file if(nError == ERROR_SUCCESS) - nError = TestFileStreamOperations("MPQ_2011_v2_EncryptedMpq.MPQE", STREAM_PROVIDER_ENCRYPTED); + nError = TestFileStreamOperations("mpqe-file://MPQ_2011_v2_EncryptedMpq.MPQE", STREAM_PROVIDER_MPQE); - // Search in listfile + // Open a stream, paired with local master + if(nError == ERROR_SUCCESS) + nError = TestReadFile_MasterMirror("MPQ_2013_v4_alternate-created.MPQ", "MPQ_2013_v4_alternate-original.MPQ", false); + + // Open a stream, paired with local master if(nError == ERROR_SUCCESS) - nError = TestReadFile_MasterMirror("MPQ_2013_v4_alternate-downloaded.MPQ", "MPQ_2013_v4_alternate-original.MPQ"); + nError = TestReadFile_MasterMirror("MPQ_2013_v4_alternate-incomplete.MPQ", "MPQ_2013_v4_alternate-incomplete.MPQ", true); + + // Open a stream, paired with local master + if(nError == ERROR_SUCCESS) + nError = TestReadFile_MasterMirror("MPQ_2013_v4_alternate-complete.MPQ", "MPQ_2013_v4_alternate-original.MPQ", true); + + // Open a stream, paired with remote master (takes hell lot of time!) + if(nError == ERROR_SUCCESS) + nError = TestReadFile_MasterMirror("MPQ_2013_v4_alternate-downloaded.MPQ", "http://www.zezula.net\\mpqs\\alternate.zip", false); // Search in listfile if(nError == ERROR_SUCCESS) @@ -3264,7 +3418,7 @@ int main(int argc, char * argv[]) if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2013_v4_patch-base-16357.MPQ"); - // Open an empty archive (found in WoW cache - it's just a header) + // Open an empty archive (A buggy MPQ with invalid HET entry count) if(nError == ERROR_SUCCESS) nError = TestOpenArchive("MPQ_2011_v4_InvalidHetEntryCount.MPQ"); @@ -3294,7 +3448,7 @@ int main(int argc, char * argv[]) // Open an encrypted archive from Starcraft II installer if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("MPQ_2011_v2_EncryptedMpq.MPQE"); + nError = TestOpenArchive("mpqe-file://MPQ_2011_v2_EncryptedMpq.MPQE"); // Open a MPK archive from Longwu online if(nError == ERROR_SUCCESS) @@ -3306,11 +3460,15 @@ int main(int argc, char * argv[]) // Open a partial MPQ with compressed hash table if(nError == ERROR_SUCCESS) - nError = TestOpenArchive("MPQ_2010_v2_HashTableCompressed.MPQ.part"); + nError = TestOpenArchive("part-file://MPQ_2010_v2_HashTableCompressed.MPQ.part"); + + // Open an archive that is merged with multiple files + if(nError == ERROR_SUCCESS) + nError = TestOpenArchive("blk4-file://streaming/model.MPQ.0"); // Open every MPQ that we have in the storage if(nError == ERROR_SUCCESS) - nError = FindFiles(ForEachFile_OpenArchive, szMpqDirectory); + nError = FindFiles(ForEachFile_OpenArchive, NULL); // Test on an archive that has been invalidated by extending an old valid MPQ if(nError == ERROR_SUCCESS) @@ -3342,7 +3500,7 @@ int main(int argc, char * argv[]) // Downloadable MPQ archive if(nError == ERROR_SUCCESS) - nError = TestOpenArchive_MasterMirror("MPQ_2013_v4_alternate-downloaded.MPQ", "MPQ_2013_v4_alternate-original.MPQ", "alternate\\DUNGEONS\\TEXTURES\\ICECROWN\\GATE\\jlo_IceC_Floor_Thrown.blp"); + nError = TestOpenArchive_MasterMirror("MPQ_2013_v4_alternate-downloaded.MPQ", "MPQ_2013_v4_alternate-original.MPQ", "alternate\\DUNGEONS\\TEXTURES\\ICECROWN\\GATE\\jlo_IceC_Floor_Thrown.blp", false); // Check archive signature if(nError == ERROR_SUCCESS) |