diff options
Diffstat (limited to 'src/FileStream.cpp')
| -rw-r--r-- | src/FileStream.cpp | 801 |
1 files changed, 617 insertions, 184 deletions
diff --git a/src/FileStream.cpp b/src/FileStream.cpp index 58543e3..c196f11 100644 --- a/src/FileStream.cpp +++ b/src/FileStream.cpp @@ -97,7 +97,7 @@ static bool BaseFile_Create(TFileStream * pStream) return true; } -static bool BaseFile_Open(TFileStream * pStream, DWORD dwStreamFlags) +static bool BaseFile_Open(TFileStream * pStream, const TCHAR * szFileName, DWORD dwStreamFlags) { #ifdef PLATFORM_WINDOWS { @@ -106,7 +106,7 @@ static bool BaseFile_Open(TFileStream * pStream, DWORD dwStreamFlags) DWORD dwWriteShare = (dwStreamFlags & STREAM_FLAG_WRITE_SHARE) ? FILE_SHARE_WRITE : 0; // Open the file - pStream->Base.File.hFile = CreateFile(pStream->szFileName, + pStream->Base.File.hFile = CreateFile(szFileName, FILE_READ_DATA | FILE_READ_ATTRIBUTES | dwWriteAccess, FILE_SHARE_READ | dwWriteShare, NULL, @@ -417,7 +417,7 @@ static void BaseFile_Init(TFileStream * pStream) //----------------------------------------------------------------------------- // Local functions - base memory-mapped file support -static bool BaseMap_Open(TFileStream * pStream, DWORD dwStreamFlags) +static bool BaseMap_Open(TFileStream * pStream, const TCHAR * szFileName, DWORD dwStreamFlags) { #ifdef PLATFORM_WINDOWS @@ -430,7 +430,7 @@ static bool BaseMap_Open(TFileStream * pStream, DWORD dwStreamFlags) dwStreamFlags = dwStreamFlags; // Open the file for read access - hFile = CreateFile(pStream->szFileName, FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); + hFile = CreateFile(szFileName, FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); if(hFile != NULL) { // Retrieve file size. Don't allow mapping file of a zero size. @@ -588,11 +588,10 @@ static const TCHAR * BaseHttp_ExtractServerName(const TCHAR * szFileName, TCHAR return szFileName; } -static bool BaseHttp_Open(TFileStream * pStream, DWORD dwStreamFlags) +static bool BaseHttp_Open(TFileStream * pStream, const TCHAR * szFileName, DWORD dwStreamFlags) { #ifdef PLATFORM_WINDOWS - const TCHAR * szFileName; HINTERNET hRequest; DWORD dwTemp = 0; bool bFileAvailable = false; @@ -624,7 +623,7 @@ static bool BaseHttp_Open(TFileStream * pStream, DWORD dwStreamFlags) DWORD dwFlags = INTERNET_FLAG_KEEP_CONNECTION | INTERNET_FLAG_NO_UI | INTERNET_FLAG_NO_CACHE_WRITE; // Initiate connection with the server - szFileName = BaseHttp_ExtractServerName(pStream->szFileName, szServerName); + szFileName = BaseHttp_ExtractServerName(szFileName, szServerName); pStream->Base.Http.hConnect = InternetConnect(pStream->Base.Http.hInternet, szServerName, INTERNET_DEFAULT_HTTP_PORT, @@ -821,11 +820,11 @@ static bool BlockStream_Read( DWORD BlockSize = pStream->BlockSize; DWORD BlockCount; bool bPrevBlockAvailable; + bool bCallbackCalled = false; bool bBlockAvailable; bool bResult = true; // The base block read function must be present - assert(pStream->BlockCheck != NULL); assert(pStream->BlockRead != NULL); // NOP reading of zero bytes @@ -837,7 +836,7 @@ static bool BlockStream_Read( EndOffset = ByteOffset + dwBytesToRead; if(EndOffset > pStream->StreamSize) { - SetLastError(ERROR_CAN_NOT_COMPLETE); + SetLastError(ERROR_HANDLE_EOF); return false; } @@ -863,6 +862,7 @@ static bool BlockStream_Read( { // Now parse the blocks and send the block read request // to all blocks with the same availability + assert(pStream->BlockCheck != NULL); bPrevBlockAvailable = pStream->BlockCheck(pStream, BlockOffset); // Loop as long as we have something to read @@ -874,6 +874,13 @@ static bool BlockStream_Read( // If the availability has changed, read all blocks up to this one if(bBlockAvailable != bPrevBlockAvailable) { + // Call the file stream callback, if the block is not available + if(pStream->pMaster && pStream->pfnCallback && bPrevBlockAvailable == false) + { + pStream->pfnCallback(pStream->UserData, BlockOffset0, (DWORD)(BlockOffset - BlockOffset0)); + bCallbackCalled = true; + } + // Load the continuous blocks with the same availability assert(BlockOffset > BlockOffset0); bResult = pStream->BlockRead(pStream, BlockOffset0, BlockOffset, BlockBuffer, BytesNeeded, bPrevBlockAvailable); @@ -894,6 +901,13 @@ static bool BlockStream_Read( // If there is a block(s) remaining to be read, do it if(BlockOffset > BlockOffset0) { + // Call the file stream callback, if the block is not available + if(pStream->pMaster && pStream->pfnCallback && bPrevBlockAvailable == false) + { + pStream->pfnCallback(pStream->UserData, BlockOffset0, (DWORD)(BlockOffset - BlockOffset0)); + bCallbackCalled = true; + } + // Read the complete blocks from the file if(BlockOffset > pStream->StreamSize) BlockOffset = pStream->StreamSize; @@ -914,6 +928,15 @@ static bool BlockStream_Read( memcpy(pvBuffer, TransferBuffer + BlockBufferOffset, dwBytesToRead); pStream->StreamPos = ByteOffset + dwBytesToRead; } + else + { + // If the block read failed, set the last error + SetLastError(ERROR_FILE_INCOMPLETE); + } + + // Call the callback to indicate we are done + if(bCallbackCalled) + pStream->pfnCallback(pStream->UserData, 0, 0); // Free the block buffer and return STORM_FREE(TransferBuffer); @@ -955,7 +978,7 @@ static STREAM_INIT StreamBaseInit[4] = }; // This function allocates an empty structure for the file stream -// The stream structure is created as variable length, linear block of data +// The stream structure is created as flat block, variable length // The file name is placed after the end of the stream structure data static TFileStream * AllocateFileStream( const TCHAR * szFileName, @@ -1015,9 +1038,9 @@ static TFileStream * AllocateFileStream( } //----------------------------------------------------------------------------- -// Local functions - linear stream support +// Local functions - flat stream support -static DWORD LinearStream_CheckFile(TBlockStream * pStream) +static DWORD FlatStream_CheckFile(TBlockStream * pStream) { LPBYTE FileBitmap = (LPBYTE)pStream->FileBitmap; DWORD WholeByteCount = (pStream->BlockCount / 8); @@ -1043,7 +1066,7 @@ static DWORD LinearStream_CheckFile(TBlockStream * pStream) return 1; } -static bool LinearStream_LoadBitmap(TBlockStream * pStream) +static bool FlatStream_LoadBitmap(TBlockStream * pStream) { FILE_BITMAP_FOOTER Footer; ULONGLONG ByteOffset; @@ -1060,7 +1083,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) { // Load the bitmap footer ByteOffset = pStream->Base.File.FileSize - sizeof(FILE_BITMAP_FOOTER); - if(pStream->StreamRead(pStream, &ByteOffset, &Footer, sizeof(FILE_BITMAP_FOOTER))) + if(pStream->BaseRead(pStream, &ByteOffset, &Footer, sizeof(FILE_BITMAP_FOOTER))) { // Make sure that the array is properly BSWAP-ed BSWAP_ARRAY32_UNSIGNED((LPDWORD)(&Footer), sizeof(FILE_BITMAP_FOOTER)); @@ -1076,7 +1099,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) // Check if the sizes match if(ByteOffset + BitmapSize + sizeof(FILE_BITMAP_FOOTER) == pStream->Base.File.FileSize) { - // Allocate space for the linear bitmap + // Allocate space for the bitmap FileBitmap = STORM_ALLOC(BYTE, BitmapSize); if(FileBitmap != NULL) { @@ -1095,7 +1118,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) pStream->BitmapSize = BitmapSize; pStream->BlockSize = Footer.BlockSize; pStream->BlockCount = BlockCount; - pStream->IsComplete = LinearStream_CheckFile(pStream); + pStream->IsComplete = FlatStream_CheckFile(pStream); return true; } } @@ -1106,7 +1129,7 @@ static bool LinearStream_LoadBitmap(TBlockStream * pStream) return false; } -static void LinearStream_UpdateBitmap( +static void FlatStream_UpdateBitmap( TBlockStream * pStream, // Pointer to an open stream ULONGLONG StartOffset, ULONGLONG EndOffset) @@ -1137,9 +1160,12 @@ static void LinearStream_UpdateBitmap( ByteIndex += (BitMask >> 0x07); BitMask = (BitMask >> 0x07) | (BitMask << 0x01); } + + // Increment the bitmap update count + pStream->IsModified = 1; } -static bool LinearStream_BlockCheck( +static bool FlatStream_BlockCheck( TBlockStream * pStream, // Pointer to an open stream ULONGLONG BlockOffset) { @@ -1159,7 +1185,7 @@ static bool LinearStream_BlockCheck( return (FileBitmap[BlockIndex / 0x08] & BitMask) ? true : false; } -static bool LinearStream_BlockRead( +static bool FlatStream_BlockRead( TBlockStream * pStream, // Pointer to an open stream ULONGLONG StartOffset, ULONGLONG EndOffset, @@ -1170,7 +1196,7 @@ static bool LinearStream_BlockRead( DWORD BytesToRead = (DWORD)(EndOffset - StartOffset); // The starting offset must be aligned to size of the block - assert(pStream->FileBitmap != NULL && pStream->IsComplete == 0); + assert(pStream->FileBitmap != NULL); assert((StartOffset & (pStream->BlockSize - 1)) == 0); assert(StartOffset < EndOffset); @@ -1190,8 +1216,8 @@ static bool LinearStream_BlockRead( // Store the loaded blocks to the mirror file. // Note that this operation is not required to succeed - if(pStream->BaseWrite(pStream, &StartOffset, BlockBuffer, BytesToRead) && bAvailable == false) - LinearStream_UpdateBitmap(pStream, StartOffset, EndOffset); + if(pStream->BaseWrite(pStream, &StartOffset, BlockBuffer, BytesToRead)) + FlatStream_UpdateBitmap(pStream, StartOffset, EndOffset); return true; } @@ -1203,16 +1229,107 @@ static bool LinearStream_BlockRead( } } -static bool LinearStream_CreateMirror(TBlockStream * pStream) +static void FlatStream_Close(TBlockStream * pStream) +{ + FILE_BITMAP_FOOTER Footer; + + if(pStream->FileBitmap && pStream->IsModified) + { + // Write the file bitmap + pStream->BaseWrite(pStream, &pStream->StreamSize, pStream->FileBitmap, pStream->BitmapSize); + + // Prepare and write the file footer + Footer.Signature = ID_FILE_BITMAP_FOOTER; + Footer.Version = 3; + Footer.BuildNumber = 10958; // BUGBUG: What build number shall we set??? + Footer.MapOffsetLo = (DWORD)(pStream->StreamSize & 0xFFFFFFFF); + Footer.MapOffsetHi = (DWORD)(pStream->StreamSize >> 0x20); + Footer.BlockSize = pStream->BlockSize; + BSWAP_ARRAY32_UNSIGNED(&Footer, sizeof(FILE_BITMAP_FOOTER)); + pStream->BaseWrite(pStream, NULL, &Footer, sizeof(FILE_BITMAP_FOOTER)); + } + + // Close the base class + BlockStream_Close(pStream); +} + +static bool FlatStream_CreateMirror(TBlockStream * pStream) { + ULONGLONG MasterSize = 0; + ULONGLONG MirrorSize = 0; + LPBYTE FileBitmap = NULL; + DWORD dwBitmapSize; + DWORD dwBlockCount; + bool bNeedCreateMirrorStream = true; + bool bNeedResizeMirrorStream = true; + // Do we have master function and base creation function? if(pStream->pMaster == NULL || pStream->BaseCreate == NULL) return false; - return false; + // Retrieve the master file size, block count and bitmap size + FileStream_GetSize(pStream->pMaster, &MasterSize); + dwBlockCount = (DWORD)((MasterSize + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE); + dwBitmapSize = (DWORD)((dwBlockCount + 7) / 8); + + // Setup stream size and position + pStream->StreamSize = MasterSize; + pStream->StreamPos = 0; + + // Open the base stream for write access + if(pStream->BaseOpen(pStream, pStream->szFileName, 0)) + { + // If the file open succeeded, check if the file size matches required size + pStream->BaseGetSize(pStream, &MirrorSize); + if(MirrorSize == MasterSize + dwBitmapSize + sizeof(FILE_BITMAP_FOOTER)) + { + // Attempt to load an existing file bitmap + if(FlatStream_LoadBitmap(pStream)) + return true; + + // We need to create new file bitmap + bNeedResizeMirrorStream = false; + } + + // We need to create mirror stream + bNeedCreateMirrorStream = false; + } + + // Create a new stream, if needed + if(bNeedCreateMirrorStream) + { + if(!pStream->BaseCreate(pStream)) + return false; + } + + // If we need to, then resize the mirror stream + if(bNeedResizeMirrorStream) + { + if(!pStream->BaseResize(pStream, MasterSize + dwBitmapSize + sizeof(FILE_BITMAP_FOOTER))) + return false; + } + + // Allocate the bitmap array + FileBitmap = STORM_ALLOC(BYTE, dwBitmapSize); + if(FileBitmap == NULL) + return false; + + // Initialize the bitmap + memset(FileBitmap, 0, dwBitmapSize); + pStream->FileBitmap = FileBitmap; + pStream->BitmapSize = dwBitmapSize; + pStream->BlockSize = DEFAULT_BLOCK_SIZE; + pStream->BlockCount = dwBlockCount; + pStream->IsComplete = 0; + pStream->IsModified = 1; + + // Note: Don't write the stream bitmap right away. + // Doing so would cause sparse file resize on NTFS, + // which would take long time on larger files. + return true; } -static TFileStream * LinearStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) +static TFileStream * FlatStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) { TBlockStream * pStream; @@ -1221,53 +1338,63 @@ static TFileStream * LinearStream_Open(const TCHAR * szFileName, DWORD dwStreamF if(pStream == NULL) return NULL; - // Attempt to open the base stream. If this fails - // and we have a master stream, we can create new mirror - assert(pStream->BaseOpen != NULL); - if(!pStream->BaseOpen(pStream, dwStreamFlags)) + // Do we have a master stream? + if(pStream->pMaster != NULL) { - // Do we have base create function and master stream? -// if(!LinearStream_CreateMirror(pStream)) + if(!FlatStream_CreateMirror(pStream)) { FileStream_Close(pStream); SetLastError(ERROR_FILE_NOT_FOUND); return NULL; } } + else + { + // Attempt to open the base stream + if(!pStream->BaseOpen(pStream, pStream->szFileName, dwStreamFlags)) + return false; - // Set the stream function as if this was linear file without bitmap - pStream->StreamRead = pStream->BaseRead; - pStream->StreamWrite = pStream->BaseWrite; - pStream->StreamResize = pStream->BaseResize; - pStream->StreamGetSize = pStream->BaseGetSize; - pStream->StreamGetPos = pStream->BaseGetPos; - pStream->StreamClose = pStream->BaseClose; + // Load the bitmap, if required to + if(dwStreamFlags & STREAM_FLAG_USE_BITMAP) + FlatStream_LoadBitmap(pStream); - // Setup the stream size - pStream->StreamSize = pStream->Base.File.FileSize; + // Setup the stream size + if(pStream->FileBitmap == NULL) + pStream->StreamSize = pStream->Base.File.FileSize; + pStream->StreamPos = 0; + } - // If we load the bitmap and find out that it's an incomplete file, - // We set the reading functions which check presence of each file block - if(LinearStream_LoadBitmap(pStream)) + // If we have a stream bitmap, set the reading functions + // which check presence of each file block + if(pStream->FileBitmap != NULL) { // Set the stream position to zero. Stream size is already set assert(pStream->StreamSize != 0); pStream->StreamPos = 0; pStream->dwFlags |= STREAM_FLAG_READ_ONLY; - // Set the reading function - if(pStream->IsComplete == 0) - { - // Supply the reading function - pStream->StreamRead = (STREAM_READ)BlockStream_Read; - pStream->StreamGetSize = BlockStream_GetSize; - pStream->StreamGetPos = BlockStream_GetPos; - pStream->StreamClose = (STREAM_CLOSE)BlockStream_Close; - - // Supply the block functions - pStream->BlockCheck = (BLOCK_CHECK)LinearStream_BlockCheck; - pStream->BlockRead = (BLOCK_READ)LinearStream_BlockRead; - } + // Supply the stream functions + pStream->StreamRead = (STREAM_READ)BlockStream_Read; + pStream->StreamGetSize = BlockStream_GetSize; + pStream->StreamGetPos = BlockStream_GetPos; + pStream->StreamClose = (STREAM_CLOSE)FlatStream_Close; + + // Supply the block functions + pStream->BlockCheck = (BLOCK_CHECK)FlatStream_BlockCheck; + pStream->BlockRead = (BLOCK_READ)FlatStream_BlockRead; + } + else + { + // Reset the file position to zero + pStream->Base.File.FilePos = 0; + + // Set the base functions + pStream->StreamRead = pStream->BaseRead; + pStream->StreamWrite = pStream->BaseWrite; + pStream->StreamResize = pStream->BaseResize; + pStream->StreamGetSize = pStream->BaseGetSize; + pStream->StreamGetPos = pStream->BaseGetPos; + pStream->StreamClose = pStream->BaseClose; } return pStream; @@ -1440,7 +1567,7 @@ static TFileStream * PartialStream_Open(const TCHAR * szFileName, DWORD dwStream // Attempt to open the base stream. If this fails // and we have a master stream, we can create new mirror assert(pStream->BaseOpen != NULL); - if(!pStream->BaseOpen(pStream, dwStreamFlags)) + if(!pStream->BaseOpen(pStream, pStream->szFileName, dwStreamFlags)) { // Do we have base create function and master stream? // if(!PartialStream_CreateMirror(pStream)) @@ -1478,7 +1605,7 @@ static TFileStream * PartialStream_Open(const TCHAR * szFileName, DWORD dwStream } //----------------------------------------------------------------------------- -// Local functions - encrypted stream support +// Local functions - MPQE stream support static const char * szKeyTemplate = "expand 32-byte k000000000000000000000000000000000000000000000000"; @@ -1672,7 +1799,7 @@ static void DecryptFileChunk( } } -static bool EncryptedStream_DetectFileKey(TEncryptedStream * pStream) +static bool MpqeStream_DetectFileKey(TEncryptedStream * pStream) { ULONGLONG ByteOffset = 0; BYTE EncryptedHeader[MPQE_CHUNK_SIZE]; @@ -1712,12 +1839,7 @@ static bool EncryptedStream_DetectFileKey(TEncryptedStream * pStream) return false; } -static bool EncryptedStream_BlockCheck(TEncryptedStream *, ULONGLONG) -{ - return true; -} - -static bool EncryptedStream_BlockRead( +static bool MpqeStream_BlockRead( TEncryptedStream * pStream, ULONGLONG StartOffset, ULONGLONG EndOffset, @@ -1731,6 +1853,7 @@ static bool EncryptedStream_BlockRead( assert(StartOffset < EndOffset); assert(bAvailable != false); BytesNeeded = BytesNeeded; + bAvailable = bAvailable; // Read the file from the stream as-is // Limit the reading to number of blocks really needed @@ -1744,67 +1867,7 @@ static bool EncryptedStream_BlockRead( return true; } -/* -static bool EncryptedStream_Read( - TEncryptedStream * pStream, // Pointer to an open stream - ULONGLONG * pByteOffset, // Pointer to file byte offset. If NULL, it reads from the current position - void * pvBuffer, // Pointer to data to be read - DWORD dwBytesToRead) // Number of bytes to read from the file -{ - ULONGLONG StartOffset; // Offset of the first byte to be read from the file - ULONGLONG ByteOffset; // Offset that the caller wants - ULONGLONG EndOffset; // End offset that is to be read from the file - DWORD dwBytesToAllocate; - DWORD dwBytesToDecrypt; - DWORD dwOffsetInCache; - LPBYTE pbMpqData = NULL; - bool bResult = false; - - // Get the byte offset - ByteOffset = (pByteOffset != NULL) ? *pByteOffset : pStream->StreamPos; - - // Cut it down to MPQE chunk size - StartOffset = ByteOffset; - StartOffset = StartOffset & ~(MPQE_CHUNK_SIZE - 1); - EndOffset = ByteOffset + dwBytesToRead; - - // Calculate number of bytes to decrypt - dwBytesToDecrypt = (DWORD)(EndOffset - StartOffset); - dwBytesToAllocate = (dwBytesToDecrypt + MPQE_CHUNK_SIZE - 1) & ~(MPQE_CHUNK_SIZE - 1); - - // Allocate buffers for encrypted and decrypted data - pbMpqData = STORM_ALLOC(BYTE, dwBytesToAllocate); - if(pbMpqData) - { - // Get the offset of the desired data in the cache - dwOffsetInCache = (DWORD)(ByteOffset - StartOffset); - - // Read the file from the stream as-is - if(pStream->BaseRead(pStream, &StartOffset, pbMpqData, dwBytesToDecrypt)) - { - // Decrypt the data - DecryptFileChunk((LPDWORD)pbMpqData, pStream->Key, StartOffset, dwBytesToAllocate); - - // Copy the decrypted data - memcpy(pvBuffer, pbMpqData + dwOffsetInCache, dwBytesToRead); - bResult = true; - } - else - { - assert(false); - } - - // Free decryption buffer - STORM_FREE(pbMpqData); - } - - // Update stream position - pStream->StreamPos = ByteOffset + dwBytesToRead; - return bResult; -} -*/ - -static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) +static TFileStream * MpqeStream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) { TEncryptedStream * pStream; @@ -1815,11 +1878,11 @@ static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStre // Attempt to open the base stream assert(pStream->BaseOpen != NULL); - if(!pStream->BaseOpen(pStream, dwStreamFlags)) + if(!pStream->BaseOpen(pStream, pStream->szFileName, dwStreamFlags)) return NULL; // Determine the encryption key for the MPQ - if(EncryptedStream_DetectFileKey(pStream)) + if(MpqeStream_DetectFileKey(pStream)) { // Set the stream position and size assert(pStream->StreamSize != 0); @@ -1833,8 +1896,7 @@ static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStre pStream->StreamClose = pStream->BaseClose; // Supply the block functions - pStream->BlockCheck = (BLOCK_CHECK)EncryptedStream_BlockCheck; - pStream->BlockRead = (BLOCK_READ)EncryptedStream_BlockRead; + pStream->BlockRead = (BLOCK_READ)MpqeStream_BlockRead; return pStream; } @@ -1845,6 +1907,203 @@ static TFileStream * EncryptedStream_Open(const TCHAR * szFileName, DWORD dwStre } //----------------------------------------------------------------------------- +// Local functions - Block4 stream support + +#define BLOCK4_BLOCK_SIZE 0x4000 // Size of one block +#define BLOCK4_HASH_SIZE 0x20 // Size of MD5 hash that is after each block +#define BLOCK4_MAX_BLOCKS 0x00002000 // Maximum amount of blocks per file +#define BLOCK4_MAX_FSIZE 0x08040000 // Max size of one file + +static bool Block4Stream_BlockRead( + TBlockStream * pStream, // Pointer to an open stream + ULONGLONG StartOffset, + ULONGLONG EndOffset, + LPBYTE BlockBuffer, + DWORD BytesNeeded, + bool bAvailable) +{ + TBaseProviderData * BaseArray = (TBaseProviderData *)pStream->FileBitmap; + ULONGLONG ByteOffset; + DWORD BytesToRead; + DWORD StreamIndex; + DWORD BlockIndex; + + // The starting offset must be aligned to size of the block + assert(pStream->FileBitmap != NULL); + assert((StartOffset & (pStream->BlockSize - 1)) == 0); + assert(StartOffset < EndOffset); + assert(bAvailable == true); + + while(BytesNeeded != 0) + { + // Calculate the block index and the file index + StreamIndex = (DWORD)((StartOffset / pStream->BlockSize) / BLOCK4_MAX_BLOCKS); + BlockIndex = (DWORD)((StartOffset / pStream->BlockSize) % BLOCK4_MAX_BLOCKS); + if(StreamIndex > pStream->BitmapSize) + return false; + + // Prepare the appropriate stream to the base variables + memcpy(&pStream->Base, BaseArray + StreamIndex, sizeof(TBaseProviderData)); + + // Calculate the block offset + ByteOffset = ((ULONGLONG)BlockIndex * (BLOCK4_BLOCK_SIZE + BLOCK4_HASH_SIZE)); + BytesToRead = min(BytesNeeded, BLOCK4_BLOCK_SIZE); + + // Read from the base stream + if(!pStream->BaseRead(pStream, &ByteOffset, BlockBuffer, BytesToRead)) + return false; + + // Move pointers + StartOffset += BytesToRead; + BlockBuffer += BytesToRead; + BytesNeeded -= BytesToRead; + } + + return true; +} + + +static void Block4Stream_Close(TBlockStream * pStream) +{ + TBaseProviderData * BaseArray = (TBaseProviderData *)pStream->FileBitmap; + + // If we have a non-zero count of base streams, + // we have to close them all + if(BaseArray != NULL) + { + // Close all base streams + for(DWORD i = 0; i < pStream->BitmapSize; i++) + { + memcpy(&pStream->Base, BaseArray + i, sizeof(TBaseProviderData)); + pStream->BaseClose(pStream); + } + } + + // Free the data map, if any + if(pStream->FileBitmap != NULL) + STORM_FREE(pStream->FileBitmap); + pStream->FileBitmap = NULL; + + // Do not call the BaseClose function, + // we closed all handles already + return; +} + +static TFileStream * Block4Stream_Open(const TCHAR * szFileName, DWORD dwStreamFlags) +{ + TBaseProviderData * NewBaseArray = NULL; + ULONGLONG RemainderBlock; + ULONGLONG BlockCount; + ULONGLONG FileSize; + TBlockStream * pStream; + TCHAR * szNameBuff; + size_t nNameLength; + DWORD dwBaseFiles = 0; + DWORD dwBaseFlags; + + // Create new empty stream + pStream = (TBlockStream *)AllocateFileStream(szFileName, sizeof(TBlockStream), dwStreamFlags); + if(pStream == NULL) + return NULL; + + // Sanity check + assert(pStream->BaseOpen != NULL); + + // Get the length of the file name without numeric suffix + nNameLength = _tcslen(pStream->szFileName); + if(pStream->szFileName[nNameLength - 2] == '.' && pStream->szFileName[nNameLength - 1] == '0') + nNameLength -= 2; + pStream->szFileName[nNameLength] = 0; + + // Supply the stream functions + pStream->StreamRead = (STREAM_READ)BlockStream_Read; + pStream->StreamGetSize = BlockStream_GetSize; + pStream->StreamGetPos = BlockStream_GetPos; + pStream->StreamClose = (STREAM_CLOSE)Block4Stream_Close; + pStream->BlockRead = (BLOCK_READ)Block4Stream_BlockRead; + + // Allocate work space for numeric names + szNameBuff = STORM_ALLOC(TCHAR, nNameLength + 4); + if(szNameBuff != NULL) + { + // Set the base flags + dwBaseFlags = (dwStreamFlags & STREAM_PROVIDERS_MASK) | STREAM_FLAG_READ_ONLY; + + // Go all suffixes from 0 to 30 + for(int nSuffix = 0; nSuffix < 30; nSuffix++) + { + // Open the n-th file + _stprintf(szNameBuff, _T("%s.%u"), pStream->szFileName, nSuffix); + if(!pStream->BaseOpen(pStream, szNameBuff, dwBaseFlags)) + break; + + // If the open succeeded, we re-allocate the base provider array + NewBaseArray = STORM_ALLOC(TBaseProviderData, dwBaseFiles + 1); + if(NewBaseArray == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return false; + } + + // Copy the old base data array to the new base data array + if(pStream->FileBitmap != NULL) + { + memcpy(NewBaseArray, pStream->FileBitmap, sizeof(TBaseProviderData) * dwBaseFiles); + STORM_FREE(pStream->FileBitmap); + } + + // Also copy the opened base array + memcpy(NewBaseArray + dwBaseFiles, &pStream->Base, sizeof(TBaseProviderData)); + pStream->FileBitmap = NewBaseArray; + dwBaseFiles++; + + // Get the size of the base stream + pStream->BaseGetSize(pStream, &FileSize); + assert(FileSize <= BLOCK4_MAX_FSIZE); + RemainderBlock = FileSize % (BLOCK4_BLOCK_SIZE + BLOCK4_HASH_SIZE); + BlockCount = FileSize / (BLOCK4_BLOCK_SIZE + BLOCK4_HASH_SIZE); + + // Increment the stream size and number of blocks + pStream->StreamSize += (BlockCount * BLOCK4_BLOCK_SIZE); + pStream->BlockCount += (DWORD)BlockCount; + + // Is this the last file? + if(FileSize < BLOCK4_MAX_FSIZE) + { + if(RemainderBlock) + { + pStream->StreamSize += (RemainderBlock - BLOCK4_HASH_SIZE); + pStream->BlockCount++; + } + break; + } + } + + // Fill the remainining block stream variables + pStream->BitmapSize = dwBaseFiles; + pStream->BlockSize = BLOCK4_BLOCK_SIZE; + pStream->IsComplete = 1; + pStream->IsModified = 0; + + // Fill the remaining stream variables + pStream->StreamPos = 0; + pStream->dwFlags |= STREAM_FLAG_READ_ONLY; + + STORM_FREE(szNameBuff); + } + + // If we opened something, return success + if(dwBaseFiles == 0) + { + FileStream_Close(pStream); + SetLastError(ERROR_FILE_NOT_FOUND); + pStream = NULL; + } + + return pStream; +} + +//----------------------------------------------------------------------------- // Public functions /** @@ -1870,14 +2129,14 @@ TFileStream * FileStream_CreateFile( { TFileStream * pStream; - // We only support creation of linear, local file - if((dwStreamFlags & (STREAM_PROVIDERS_MASK)) != (STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE)) + // We only support creation of flat, local file + if((dwStreamFlags & (STREAM_PROVIDERS_MASK)) != (STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE)) { SetLastError(ERROR_NOT_SUPPORTED); return NULL; } - // Allocate file stream structure for linear stream + // Allocate file stream structure for flat stream pStream = AllocateFileStream(szFileName, sizeof(TBlockStream), dwStreamFlags); if(pStream != NULL) { @@ -1924,41 +2183,27 @@ TFileStream * FileStream_OpenFile( const TCHAR * szFileName, DWORD dwStreamFlags) { - DWORD dwBaseProvider = dwStreamFlags & BASE_PROVIDER_MASK; - - // The "file:" prefix forces the BASE_PROVIDER_FILE - if(!_tcsicmp(szFileName, _T("file:"))) - { - dwBaseProvider = BASE_PROVIDER_FILE; - szFileName += 5; - } - - // The "map:" prefix forces the BASE_PROVIDER_MAP - if(!_tcsicmp(szFileName, _T("map:"))) - { - dwBaseProvider = BASE_PROVIDER_MAP; - szFileName += 4; - } - - // The "http:" prefix forces the BASE_PROVIDER_HTTP - if(!_tcsicmp(szFileName, _T("http:"))) - { - dwBaseProvider = BASE_PROVIDER_HTTP; - szFileName += 5; - } + DWORD dwProvider = dwStreamFlags & STREAM_PROVIDERS_MASK; + size_t nPrefixLength = FileStream_Prefix(szFileName, &dwProvider); // Re-assemble the stream flags - dwStreamFlags = (dwStreamFlags & STREAM_OPTIONS_MASK) | (dwStreamFlags & STREAM_PROVIDER_MASK) | dwBaseProvider; + dwStreamFlags = (dwStreamFlags & STREAM_OPTIONS_MASK) | dwProvider; + szFileName += nPrefixLength; + + // Perform provider-specific open switch(dwStreamFlags & STREAM_PROVIDER_MASK) { - case STREAM_PROVIDER_LINEAR: - return LinearStream_Open(szFileName, dwStreamFlags); + case STREAM_PROVIDER_FLAT: + return FlatStream_Open(szFileName, dwStreamFlags); case STREAM_PROVIDER_PARTIAL: return PartialStream_Open(szFileName, dwStreamFlags); - case STREAM_PROVIDER_ENCRYPTED: - return EncryptedStream_Open(szFileName, dwStreamFlags); + case STREAM_PROVIDER_MPQE: + return MpqeStream_Open(szFileName, dwStreamFlags); + + case STREAM_PROVIDER_BLOCK4: + return Block4Stream_Open(szFileName, dwStreamFlags); default: SetLastError(ERROR_INVALID_PARAMETER); @@ -1967,6 +2212,215 @@ TFileStream * FileStream_OpenFile( } /** + * Returns the file name of the stream + * + * \a pStream Pointer to an open stream + */ +const TCHAR * FileStream_GetFileName(TFileStream * pStream) +{ + assert(pStream != NULL); + return pStream->szFileName; +} + +/** + * Returns the length of the provider prefix. Returns zero if no prefix + * + * \a szFileName Pointer to a stream name (file, mapped file, URL) + * \a pdwStreamProvider Pointer to a DWORD variable that receives stream provider (STREAM_PROVIDER_XXX) + */ + +size_t FileStream_Prefix(const TCHAR * szFileName, DWORD * pdwProvider) +{ + size_t nPrefixLength1 = 0; + size_t nPrefixLength2 = 0; + DWORD dwProvider = 0; + + if(szFileName != NULL) + { + // + // Determine the stream provider + // + + if(!_tcsnicmp(szFileName, _T("flat-"), 5)) + { + dwProvider |= STREAM_PROVIDER_FLAT; + nPrefixLength1 = 5; + } + + if(!_tcsnicmp(szFileName, _T("part-"), 5)) + { + dwProvider |= STREAM_PROVIDER_PARTIAL; + nPrefixLength1 = 5; + } + + if(!_tcsnicmp(szFileName, _T("mpqe-"), 5)) + { + dwProvider |= STREAM_PROVIDER_MPQE; + nPrefixLength1 = 5; + } + + if(!_tcsnicmp(szFileName, _T("blk4-"), 5)) + { + dwProvider |= STREAM_PROVIDER_BLOCK4; + nPrefixLength1 = 5; + } + + // + // Determine the base provider + // + + if(!_tcsnicmp(szFileName+nPrefixLength1, _T("file:"), 5)) + { + dwProvider |= BASE_PROVIDER_FILE; + nPrefixLength2 = 5; + } + + if(!_tcsnicmp(szFileName+nPrefixLength1, _T("map:"), 4)) + { + dwProvider |= BASE_PROVIDER_MAP; + nPrefixLength2 = 4; + } + + if(!_tcsnicmp(szFileName+nPrefixLength1, _T("http:"), 5)) + { + dwProvider |= BASE_PROVIDER_HTTP; + nPrefixLength2 = 5; + } + + // Only accept stream provider if we recognized the base provider + if(nPrefixLength2 != 0) + { + // It is also allowed to put "//" after the base provider, e.g. "file://", "http://" + if(szFileName[nPrefixLength1+nPrefixLength2] == '/' && szFileName[nPrefixLength1+nPrefixLength2+1] == '/') + nPrefixLength2 += 2; + + if(pdwProvider != NULL) + *pdwProvider = dwProvider; + return nPrefixLength1 + nPrefixLength2; + } + } + + return 0; +} + +/** + * Sets a download callback. Whenever the stream needs to download one or more blocks + * from the server, the callback is called + * + * \a pStream Pointer to an open stream + * \a pfnCallback Pointer to callback function + * \a pvUserData Arbitrary user pointer passed to the download callback + */ + +bool FileStream_SetCallback(TFileStream * pStream, SFILE_DOWNLOAD_CALLBACK pfnCallback, void * pvUserData) +{ + TBlockStream * pBlockStream = (TBlockStream *)pStream; + + if(pStream->BlockRead == NULL) + { + SetLastError(ERROR_NOT_SUPPORTED); + return false; + } + + pBlockStream->pfnCallback = pfnCallback; + pBlockStream->UserData = pvUserData; + return true; +} + +/** + * This function gives the block map. The 'pvBitmap' pointer must point to a buffer + * of at least sizeof(STREAM_BLOCK_MAP) size. It can also have size of the complete + * block map (i.e. sizeof(STREAM_BLOCK_MAP) + BitmapSize). In that case, the function + * also copies the bit-based block map. + * + * \a pStream Pointer to an open stream + * \a pvBitmap Pointer to buffer where the block map will be stored + * \a cbBitmap Length of the buffer, of the block map + * \a cbLengthNeeded Length of the bitmap, in bytes + */ + +bool FileStream_GetBitmap(TFileStream * pStream, void * pvBitmap, DWORD cbBitmap, PDWORD pcbLengthNeeded) +{ + TStreamBitmap * pBitmap = (TStreamBitmap *)pvBitmap; + TBlockStream * pBlockStream = (TBlockStream *)pStream; + ULONGLONG BlockOffset; + LPBYTE Bitmap = (LPBYTE)(pBitmap + 1); + DWORD BitmapSize; + DWORD BlockCount; + DWORD BlockSize; + bool bResult = false; + + // Retrieve the size of one block + if(pStream->BlockCheck != NULL) + { + BlockCount = pBlockStream->BlockCount; + BlockSize = pBlockStream->BlockSize; + } + else + { + BlockCount = (DWORD)((pStream->StreamSize + DEFAULT_BLOCK_SIZE - 1) / DEFAULT_BLOCK_SIZE); + BlockSize = DEFAULT_BLOCK_SIZE; + } + + // Fill-in the variables + BitmapSize = (BlockCount + 7) / 8; + + // Give the number of blocks + if(pcbLengthNeeded != NULL) + pcbLengthNeeded[0] = sizeof(TStreamBitmap) + BitmapSize; + + // If the length of the buffer is not enough + if(pvBitmap != NULL && cbBitmap != 0) + { + // Give the STREAM_BLOCK_MAP structure + if(cbBitmap >= sizeof(TStreamBitmap)) + { + pBitmap->StreamSize = pStream->StreamSize; + pBitmap->BitmapSize = BitmapSize; + pBitmap->BlockCount = BlockCount; + pBitmap->BlockSize = BlockSize; + pBitmap->IsComplete = (pStream->BlockCheck != NULL) ? pBlockStream->IsComplete : 1; + bResult = true; + } + + // Give the block bitmap, if enough space + if(cbBitmap >= sizeof(TStreamBitmap) + BitmapSize) + { + // Version with bitmap present + if(pStream->BlockCheck != NULL) + { + DWORD ByteIndex = 0; + BYTE BitMask = 0x01; + + // Initialize the map with zeros + memset(Bitmap, 0, BitmapSize); + + // Fill the map + for(BlockOffset = 0; BlockOffset < pStream->StreamSize; BlockOffset += BlockSize) + { + // Set the bit if the block is present + if(pBlockStream->BlockCheck(pStream, BlockOffset)) + Bitmap[ByteIndex] |= BitMask; + + // Move bit position + ByteIndex += (BitMask >> 0x07); + BitMask = (BitMask >> 0x07) | (BitMask << 0x01); + } + } + else + { + memset(Bitmap, 0xFF, BitmapSize); + } + } + } + + // Set last error value and return + if(bResult == false) + SetLastError(ERROR_INSUFFICIENT_BUFFER); + return bResult; +} + +/** * Reads data from the stream * * - Returns true if the read operation succeeded and all bytes have been read @@ -2094,8 +2548,8 @@ bool FileStream_GetFlags(TFileStream * pStream, LPDWORD pdwStreamFlags) */ bool FileStream_Replace(TFileStream * pStream, TFileStream * pNewStream) { - // Only supported on linear files - if((pStream->dwFlags & STREAM_PROVIDERS_MASK) != (STREAM_PROVIDER_LINEAR | BASE_PROVIDER_FILE)) + // Only supported on flat files + if((pStream->dwFlags & STREAM_PROVIDERS_MASK) != (STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE)) { SetLastError(ERROR_NOT_SUPPORTED); return false; @@ -2117,7 +2571,7 @@ bool FileStream_Replace(TFileStream * pStream, TFileStream * pNewStream) return false; // Now open the base file again - if(BaseFile_Open(pStream, pStream->dwFlags)) + if(BaseFile_Open(pStream, pStream->szFileName, pStream->dwFlags)) return false; // Cleanup the new stream @@ -2126,27 +2580,6 @@ bool FileStream_Replace(TFileStream * pStream, TFileStream * pNewStream) } /** - * Returns the file name of the stream - * - * \a pStream Pointer to an open stream - */ -const TCHAR * FileStream_GetFileName(TFileStream * pStream) -{ - assert(pStream != NULL); - return pStream->szFileName; -} - -/** - * Returns true if the stream is read-only - * - * \a pStream Pointer to an open stream - */ -bool FileStream_IsReadOnly(TFileStream * pStream) -{ - return (pStream->dwFlags & STREAM_FLAG_READ_ONLY) ? true : false; -} - -/** * This function closes an archive file and frees any data buffers * that have been allocated for stream management. The function must also * support partially allocated structure, i.e. one or more buffers |