From fc330fd8ff0115804d9c4b53a1f810c00dd63de9 Mon Sep 17 00:00:00 2001 From: Shauren Date: Thu, 6 Jun 2019 16:48:21 +0200 Subject: Dep/CascLib: Update to ladislav-zezula/CascLib@a1197edf0b3bd4d52c3f39be7fa7b44bb0b98012 --- dep/CascLib/src/CascReadFile.cpp | 913 ++++++++++++++++++++++++--------------- 1 file changed, 564 insertions(+), 349 deletions(-) (limited to 'dep/CascLib/src/CascReadFile.cpp') diff --git a/dep/CascLib/src/CascReadFile.cpp b/dep/CascLib/src/CascReadFile.cpp index 1b13e873f02..2720d6d5adb 100644 --- a/dep/CascLib/src/CascReadFile.cpp +++ b/dep/CascLib/src/CascReadFile.cpp @@ -1,4 +1,4 @@ -/*****************************************************************************/ +/****************************************************************************/ /* CascOpenFile.cpp Copyright (c) Ladislav Zezula 2014 */ /*---------------------------------------------------------------------------*/ /* System-dependent directory functions for CascLib */ @@ -12,303 +12,454 @@ #include "CascLib.h" #include "CascCommon.h" -//----------------------------------------------------------------------------- -// Local structures - -typedef struct _BLTE_FRAME -{ - BYTE CompressedSize[4]; // Compressed file size as big endian - BYTE FrameSize[4]; // File size as big endian - BYTE md5[MD5_HASH_SIZE]; // Hash of the compressed frame - -} BLTE_FRAME, *PBLTE_FRAME; - //----------------------------------------------------------------------------- // Local functions -TCascFile * IsValidFileHandle(HANDLE hFile); // In CascOpenFile.cpp - static int EnsureDataStreamIsOpen(TCascFile * hf) { TCascStorage * hs = hf->hs; TFileStream * pStream = NULL; + ULONGLONG EncodedSize = 0; TCHAR * szDataFile; - TCHAR szPlainName[0x40]; + TCHAR szCachePath[MAX_PATH]; + TCHAR szPlainName[0x80]; + int nError; - // If the file is not open yet, do it - if(hs->DataFileArray[hf->ArchiveIndex] == NULL) + // If the file is available locally, we rely on data files. + // If not, we download the file and open the stream + if(hf->pCKeyEntry->Flags & CASC_CE_FILE_IS_LOCAL) { - // Prepare the name of the data file - _stprintf(szPlainName, _T("data.%03u"), hf->ArchiveIndex); - szDataFile = CombinePath(hs->szIndexPath, szPlainName); + // If the file is not open yet, do it + if(hs->DataFiles[hf->ArchiveIndex] == NULL) + { + // Prepare the name of the data file + CascStrPrintf(szPlainName, _countof(szPlainName), _T("data.%03u"), hf->ArchiveIndex); + szDataFile = CombinePath(hs->szIndexPath, szPlainName); - // Open the data file - if(szDataFile != NULL) + // Open the data file + if(szDataFile != NULL) + { + // Open the data stream with read+write sharing to prevent Battle.net agent + // detecting a corruption and redownloading the entire package + pStream = FileStream_OpenFile(szDataFile, STREAM_FLAG_READ_ONLY | STREAM_FLAG_WRITE_SHARE | STREAM_PROVIDER_FLAT | STREAM_FLAG_FILL_MISSING | BASE_PROVIDER_FILE); + hs->DataFiles[hf->ArchiveIndex] = pStream; + CASC_FREE(szDataFile); + } + } + + // Return error or success + hf->pStream = hs->DataFiles[hf->ArchiveIndex]; + return (hf->pStream != NULL) ? ERROR_SUCCESS : ERROR_FILE_NOT_FOUND; + } + else + { + if(hf->bDownloadFileIf) { - // Open the stream - pStream = FileStream_OpenFile(szDataFile, STREAM_FLAG_READ_ONLY | STREAM_PROVIDER_FLAT | BASE_PROVIDER_FILE); - hs->DataFileArray[hf->ArchiveIndex] = pStream; - CASC_FREE(szDataFile); + // Create the local folder path and download the file from CDN + nError = DownloadFileFromCDN(hf->hs, _T("data"), hf->pCKeyEntry->EKey, NULL, szCachePath, _countof(szCachePath)); + if(nError == ERROR_SUCCESS) + { + hf->pStream = FileStream_OpenFile(szCachePath, BASE_PROVIDER_FILE | STREAM_PROVIDER_FLAT); + if(hf->pStream != NULL) + { + // Supply the file size, if unknown yet + if(hf->EncodedSize == CASC_INVALID_SIZE) + { + FileStream_GetSize(hf->pStream, &EncodedSize); + hf->pCKeyEntry->EncodedSize = (DWORD)EncodedSize; + hf->EncodedSize = (DWORD)EncodedSize; + } + + hf->bLocalFileStream = true; + return ERROR_SUCCESS; + } + } } + + return ERROR_FILE_OFFLINE; } +} - // Return error or success - hf->pStream = hs->DataFileArray[hf->ArchiveIndex]; - return (hf->pStream != NULL) ? ERROR_SUCCESS : ERROR_FILE_NOT_FOUND; +#ifdef _DEBUG +static unsigned int table_16C57A8[0x10] = +{ + 0x049396B8, 0x72A82A9B, 0xEE626CCA, 0x9917754F, + 0x15DE40B1, 0xF5A8A9B6, 0x421EAC7E, 0xA9D55C9A, + 0x317FD40C, 0x04FAF80D, 0x3D6BE971, 0x52933CFD, + 0x27F64B7D, 0xC6F5C11B, 0xD5757E3A, 0x6C388745 +}; + +// Obtained from Agent.exe v 2.15.0.6296 (d14ec9d9a1b396a42964b05f40ea55f37eae5478d550c07ebb6cb09e50968d62) +// Note the "Checksum" value probably won't match with older game versions. +static void VerifyHeaderSpan(PBLTE_ENCODED_HEADER pBlteHeader, ULONGLONG HeaderOffset) +{ + LPBYTE pbBlteHeader = (LPBYTE)pBlteHeader; + DWORD dwInt32; + BYTE EncodedOffset[4] = { 0 }; + BYTE HashedHeader[4] = { 0 }; + BYTE JenkinsHash[4]; + BYTE Checksum[4]; + size_t i, j; + + // Seems to be hardcoded to zero + assert(pBlteHeader->field_15 == 0); + + // Calculate the Jenkins hash and write it to the header + dwInt32 = hashlittle(pbBlteHeader, FIELD_OFFSET(BLTE_ENCODED_HEADER, JenkinsHash), 0x3D6BE971); + ConvertIntegerToBytes_4_LE(dwInt32, JenkinsHash); +// assert(memcmp(pBlteHeader->JenkinsHash, JenkinsHash, sizeof(JenkinsHash)) == 0); + + // Encode the lower 32-bits of the offset + dwInt32 = (DWORD)(HeaderOffset + FIELD_OFFSET(BLTE_ENCODED_HEADER, Signature)); + dwInt32 = table_16C57A8[dwInt32 & 0x0F] ^ dwInt32; + ConvertIntegerToBytes_4_LE(dwInt32, EncodedOffset); + + // Calculate checksum of the so-far filled structure + for (i = 0; i < FIELD_OFFSET(BLTE_ENCODED_HEADER, Checksum); i++) + HashedHeader[i & 3] ^= pbBlteHeader[i]; + + // XOR the two values together to get the final checksum. + for (j = 0; j < 4; j++, i++) + Checksum[j] = HashedHeader[i & 3] ^ EncodedOffset[i & 3]; +// assert(memcmp(pBlteHeader->Checksum, Checksum, sizeof(Checksum)) == 0); } +#endif -static int LoadFileFrames(TCascFile * hf) +static int ParseBlteHeader(TCascFile * hf, ULONGLONG HeaderOffset, LPBYTE pbEncodedBuffer, size_t cbEncodedBuffer, size_t * pcbHeaderSize) +{ + PBLTE_ENCODED_HEADER pEncodedHeader = (PBLTE_ENCODED_HEADER)pbEncodedBuffer; + PBLTE_HEADER pBlteHeader = (PBLTE_HEADER)pbEncodedBuffer; + DWORD ExpectedHeaderSize; + DWORD ExHeaderSize = 0; + DWORD HeaderSize; + DWORD FrameCount = 0; + + CASCLIB_UNUSED(HeaderOffset); + + // On files within storage segments ("data.###"), there is BLTE_ENCODED_HEADER + // On local files, there is just PBLTE_HEADER + if(ConvertBytesToInteger_4_LE(pBlteHeader->Signature) != BLTE_HEADER_SIGNATURE) + { + // There must be at least some bytes + if (cbEncodedBuffer < FIELD_OFFSET(BLTE_ENCODED_HEADER, MustBe0F)) + return ERROR_BAD_FORMAT; + if (pEncodedHeader->EncodedSize != hf->EncodedSize) + return ERROR_BAD_FORMAT; + +#ifdef _DEBUG + // Not really needed, it's here just for explanation of what the values mean + //assert(memcmp(hf->pCKeyEntry->EKey, pEncodedHeader->EKey.Value, MD5_HASH_SIZE) == 0); + VerifyHeaderSpan(pEncodedHeader, HeaderOffset); +#endif + // Capture the EKey + ExHeaderSize = FIELD_OFFSET(BLTE_ENCODED_HEADER, Signature); + pBlteHeader = (PBLTE_HEADER)(pbEncodedBuffer + ExHeaderSize); + } + + // Verify the signature + if(ConvertBytesToInteger_4_LE(pBlteHeader->Signature) != BLTE_HEADER_SIGNATURE) + return ERROR_BAD_FORMAT; + + // Capture the header size. If this is non-zero, then array + // of chunk headers follow. Otherwise, the file is just one chunk + HeaderSize = ConvertBytesToInteger_4(pBlteHeader->HeaderSize); + if (HeaderSize != 0) + { + if (pBlteHeader->MustBe0F != 0x0F) + return ERROR_BAD_FORMAT; + + // Verify the header size + FrameCount = ConvertBytesToInteger_3(pBlteHeader->FrameCount); + ExpectedHeaderSize = 0x0C + FrameCount * sizeof(BLTE_FRAME); + if (ExpectedHeaderSize != HeaderSize) + return ERROR_BAD_FORMAT; + + // Give the values + pcbHeaderSize[0] = ExHeaderSize + FIELD_OFFSET(BLTE_HEADER, MustBe0F) + sizeof(DWORD); + } + else + { + pcbHeaderSize[0] = ExHeaderSize + FIELD_OFFSET(BLTE_HEADER, MustBe0F); + } + + // Give the frame count + hf->FrameCount = FrameCount; + return ERROR_SUCCESS; +} + +static LPBYTE ReadMissingHeaderData(TCascFile * hf, ULONGLONG DataFileOffset, LPBYTE pbEncodedBuffer, size_t cbEncodedBuffer, size_t cbTotalHeaderSize) +{ + LPBYTE pbNewBuffer; + + // Reallocate the buffer + pbNewBuffer = CASC_REALLOC(BYTE, pbEncodedBuffer, cbTotalHeaderSize); + if (pbNewBuffer != NULL) + { + // Load the missing data + DataFileOffset += cbEncodedBuffer; + if (FileStream_Read(hf->pStream, &DataFileOffset, pbNewBuffer + cbEncodedBuffer, (DWORD)(cbTotalHeaderSize - cbEncodedBuffer))) + { + return pbNewBuffer; + } + } + + // If anything failed, we free the original buffer and return NULL; + CASC_FREE(pbEncodedBuffer); + return NULL; +} + +static int LoadFileFrames(TCascFile * hf, ULONGLONG DataFileOffset, LPBYTE pbFramePtr, LPBYTE pbFrameEnd, size_t cbHeaderSize) { - PBLTE_FRAME pFileFrames; PBLTE_FRAME pFileFrame; - ULONGLONG ArchiveFileOffset; - DWORD FrameOffset = 0; - DWORD FileSize = 0; + DWORD ContentSize = 0; + DWORD FileOffset = 0; int nError = ERROR_SUCCESS; assert(hf != NULL); assert(hf->pStream != NULL); - assert(hf->pFrames != NULL); + assert(hf->pFrames == NULL); - // Allocate frame array - pFileFrames = pFileFrame = CASC_ALLOC(BLTE_FRAME, hf->FrameCount); - if(pFileFrames != NULL) + if (hf->FrameCount != 0) { - // Load the frame array - ArchiveFileOffset = hf->FramesOffset; - if(FileStream_Read(hf->pStream, &ArchiveFileOffset, pFileFrames, hf->FrameCount * sizeof(BLTE_FRAME))) - { - // Move the raw archive offset - ArchiveFileOffset += (hf->FrameCount * sizeof(BLTE_FRAME)); + // Move the raw archive offset + DataFileOffset += (hf->FrameCount * sizeof(BLTE_FRAME)); + // Allocate array of file frames + hf->pFrames = CASC_ALLOC(CASC_FILE_FRAME, hf->FrameCount); + if (hf->pFrames != NULL) + { // Copy the frames to the file structure - for(DWORD i = 0; i < hf->FrameCount; i++, pFileFrame++) + for (DWORD i = 0; i < hf->FrameCount; i++, pbFramePtr += sizeof(BLTE_FRAME)) { - hf->pFrames[i].FrameArchiveOffset = (DWORD)ArchiveFileOffset; - hf->pFrames[i].FrameFileOffset = FrameOffset; - hf->pFrames[i].CompressedSize = ConvertBytesToInteger_4(pFileFrame->CompressedSize); - hf->pFrames[i].FrameSize = ConvertBytesToInteger_4(pFileFrame->FrameSize); - memcpy(hf->pFrames[i].md5, pFileFrame->md5, MD5_HASH_SIZE); - - ArchiveFileOffset += hf->pFrames[i].CompressedSize; - FrameOffset += hf->pFrames[i].FrameSize; - FileSize += hf->pFrames[i].FrameSize; + // Capture the file frame + if ((pbFramePtr + sizeof(BLTE_FRAME)) > pbFrameEnd) + return ERROR_BAD_FORMAT; + pFileFrame = (PBLTE_FRAME)pbFramePtr; + + // Convert the file frame to the native format + hf->pFrames[i].DataFileOffset = (DWORD)DataFileOffset; + hf->pFrames[i].FileOffset = CASC_INVALID_POS; + hf->pFrames[i].EncodedSize = ConvertBytesToInteger_4(pFileFrame->EncodedSize); + hf->pFrames[i].ContentSize = ConvertBytesToInteger_4(pFileFrame->ContentSize); + hf->pFrames[i].FrameHash = pFileFrame->FrameHash; + + DataFileOffset += hf->pFrames[i].EncodedSize; + ContentSize += hf->pFrames[i].ContentSize; + FileOffset += hf->pFrames[i].ContentSize; } - } - else - nError = GetLastError(); - // Note: on ENCODING file, this value is almost always bigger - // then the real size of ENCODING. We handle this problem - // by calculating size of the ENCODIG file from its header. - hf->FileSize = FileSize; + // Save the content size of the file + if(hf->pCKeyEntry->ContentSize == CASC_INVALID_SIZE) + { + hf->pCKeyEntry->ContentSize = ContentSize; + hf->ContentSize = ContentSize; + } + } + } + else + { + // The content size in the file structure must be valid at this point, + // otherwise we don't know the frame content size + if (hf->ContentSize == CASC_INVALID_SIZE) + { + assert(false); + return ERROR_CAN_NOT_COMPLETE; + } -#ifdef CASCLIB_TEST - hf->FileSize_FrameSum = FileSize; -#endif + // Save the number of file frames + hf->FrameCount = 1; - // Free the array - CASC_FREE(pFileFrames); + // Allocate single "dummy" frame + hf->pFrames = CASC_ALLOC(CASC_FILE_FRAME, 1); + if (hf->pFrames != NULL) + { + memset(&hf->pFrames->FrameHash, 0, sizeof(CONTENT_KEY)); + hf->pFrames->DataFileOffset = (DWORD)DataFileOffset; + hf->pFrames->FileOffset = CASC_INVALID_POS; + hf->pFrames->EncodedSize = (DWORD)(hf->EncodedSize - cbHeaderSize); + hf->pFrames->ContentSize = hf->ContentSize; + } } - else - nError = ERROR_NOT_ENOUGH_MEMORY; + if (hf->pFrames == NULL) + nError = ERROR_NOT_ENOUGH_MEMORY; return nError; } -static int EnsureHeaderAreaIsLoaded(TCascFile * hf) +static int LoadEncodedHeaderAndFileFrames(TCascFile * hf) { - TCascStorage * hs = hf->hs; - ULONGLONG FileOffset = hf->HeaderOffset; - LPBYTE pbHeaderArea; - DWORD FileSignature; - DWORD FileSize; - BYTE HeaderArea[MAX_HEADER_AREA_SIZE]; - int nError; - - // We need the data file to be open - nError = EnsureDataStreamIsOpen(hf); - if(nError != ERROR_SUCCESS) - return nError; - - // Make sure that we already know the shift - // to the begin of file data. - // Note that older builds of Heroes of the Storm have entries pointing - // to the beginning of the header area. - // Newer versions of HOTS have encoding entries pointing directly to - // the BLTE header - if(hs->dwFileBeginDelta == 0xFFFFFFFF) - { - FileSignature = 0; - FileOffset = hf->HeaderOffset; - if(!FileStream_Read(hf->pStream, &FileOffset, &FileSignature, sizeof(DWORD))) - return ERROR_FILE_CORRUPT; + LPBYTE pbEncodedBuffer; + size_t cbEncodedBuffer = MAX_ENCODED_HEADER; + int nError = ERROR_SUCCESS; - hs->dwFileBeginDelta = (FileSignature == BLTE_HEADER_SIGNATURE) ? BLTE_HEADER_DELTA : 0; - } + // Should only be called when the file frames are NOT loaded + assert(hf->pFrames == NULL); + assert(hf->FrameCount == 0); - // If the file size is not loaded yet, do it - if(hf->FrameCount == 0) + // Allocate the initial buffer for the encoded headers + pbEncodedBuffer = CASC_ALLOC(BYTE, MAX_ENCODED_HEADER); + if (pbEncodedBuffer != NULL) { - // Load the part before BLTE header + header itself - FileOffset = hf->HeaderOffset - hs->dwFileBeginDelta; - if(!FileStream_Read(hf->pStream, &FileOffset, HeaderArea, sizeof(HeaderArea))) - return ERROR_FILE_CORRUPT; - - // Copy the MD5 hash of the frame array - memcpy(hf->FrameArrayHash, HeaderArea, MD5_HASH_SIZE); - pbHeaderArea = HeaderArea + MD5_HASH_SIZE; - - // Copy the file size - FileSize = ConvertBytesToInteger_4_LE(pbHeaderArea); - pbHeaderArea += 0x0E; - - // Verify the BLTE signature - if(ConvertBytesToInteger_4_LE(pbHeaderArea) != BLTE_HEADER_SIGNATURE) - return ERROR_BAD_FORMAT; - pbHeaderArea += sizeof(DWORD); + ULONGLONG ReadOffset = hf->ArchiveOffset; + size_t cbTotalHeaderSize; + size_t cbHeaderSize = 0; - // Load the size of the frame headers - hf->HeaderSize = ConvertBytesToInteger_4(pbHeaderArea); - if(hf->HeaderSize & 0x80000000) - return ERROR_BAD_FORMAT; - pbHeaderArea += sizeof(DWORD); + // At this point, we expect encoded size to be known + assert(hf->EncodedSize != CASC_INVALID_SIZE); - // Read the header size - assert(hs->dwFileBeginDelta <= BLTE_HEADER_DELTA); - hf->HeaderOffset += (BLTE_HEADER_DELTA - hs->dwFileBeginDelta); - hf->FrameCount = 1; + // Do not read more than encoded size + cbEncodedBuffer = CASCLIB_MIN(cbEncodedBuffer, hf->EncodedSize); - // Retrieve the frame count, if different from 1 - if(hf->HeaderSize != 0) + // Load the entire (eventual) header area. This is faster than doing + // two read operations in a row. Read as much as possible. If the file is cut, + // the FileStream will pad it with zeros + if (FileStream_Read(hf->pStream, &ReadOffset, pbEncodedBuffer, (DWORD)cbEncodedBuffer)) { - // The next byte must be 0x0F - if(pbHeaderArea[0] != 0x0F) - return ERROR_BAD_FORMAT; - pbHeaderArea++; + // Parse the BLTE header + nError = ParseBlteHeader(hf, ReadOffset, pbEncodedBuffer, cbEncodedBuffer, &cbHeaderSize); + if (nError == ERROR_SUCCESS) + { + // If the headers are larger than the initial read size, + // We read the missing data + cbTotalHeaderSize = cbHeaderSize + (hf->FrameCount * sizeof(BLTE_FRAME)); + if (cbTotalHeaderSize > cbEncodedBuffer) + { + pbEncodedBuffer = ReadMissingHeaderData(hf, ReadOffset, pbEncodedBuffer, cbEncodedBuffer, cbTotalHeaderSize); + if (pbEncodedBuffer == NULL) + nError = GetLastError(); + cbEncodedBuffer = cbTotalHeaderSize; + } - // Next three bytes form number of frames - hf->FrameCount = ConvertBytesToInteger_3(pbHeaderArea); + // Load the array of frame headers + if (nError == ERROR_SUCCESS) + { + nError = LoadFileFrames(hf, ReadOffset + cbHeaderSize, pbEncodedBuffer + cbHeaderSize, pbEncodedBuffer + cbEncodedBuffer, cbHeaderSize); + } + } + } + else + { + nError = ERROR_FILE_CORRUPT; } -#ifdef CASCLIB_TEST - hf->FileSize_HdrArea = FileSize; -#endif + // Free the frame buffer + CASC_FREE(pbEncodedBuffer); + } + else + { + nError = ERROR_NOT_ENOUGH_MEMORY; } - return ERROR_SUCCESS; + return nError; } -static int EnsureFrameHeadersLoaded(TCascFile * hf) +static int EnsureFileFramesLoaded(TCascFile * hf) { - int nError; - - // Make sure we have header area loaded - nError = EnsureHeaderAreaIsLoaded(hf); - if(nError != ERROR_SUCCESS) - return nError; + int nError = ERROR_SUCCESS; - // If the frame headers are not loaded yet, do it + // If the encoded frames are not loaded, do it now if(hf->pFrames == NULL) { - // Allocate the frame array - hf->pFrames = CASC_ALLOC(CASC_FILE_FRAME, hf->FrameCount); - if(hf->pFrames != NULL) - { - // Either load the frames from the file or supply them on our own - if(hf->HeaderSize != 0) - { - hf->FramesOffset = hf->HeaderOffset + sizeof(DWORD) + sizeof(DWORD) + sizeof(DWORD); - nError = LoadFileFrames(hf); - } - else - { - // Offset of the first frame is right after the file frames - hf->FramesOffset = hf->HeaderOffset + sizeof(DWORD) + sizeof(DWORD); - - hf->pFrames[0].FrameArchiveOffset = hf->FramesOffset; - hf->pFrames[0].FrameFileOffset = 0; - hf->pFrames[0].CompressedSize = hf->CompressedSize; - hf->pFrames[0].FrameSize = hf->FileSize; - memset(hf->pFrames[0].md5, 0, MD5_HASH_SIZE); - } - } + // We need the data file to be open + nError = EnsureDataStreamIsOpen(hf); + if(nError != ERROR_SUCCESS) + return nError; - // Return result - return (hf->pFrames != NULL) ? ERROR_SUCCESS : ERROR_FILE_CORRUPT; + // Make sure we have header area loaded + nError = LoadEncodedHeaderAndFileFrames(hf); } - return ERROR_SUCCESS; + return nError; } -static PCASC_FILE_FRAME FindFileFrame(TCascFile * hf, DWORD FilePointer) +static int LoadEncodedFrame(TFileStream * pStream, PCASC_FILE_FRAME pFrame, LPBYTE pbEncodedFrame, bool bVerifyIntegrity) { - PCASC_FILE_FRAME pFrame = hf->pFrames; - DWORD FrameBegin; - DWORD FrameEnd; - - // Sanity checks - assert(hf->pFrames != NULL); - assert(hf->FrameCount != 0); + ULONGLONG FileOffset = pFrame->DataFileOffset; + int nError = ERROR_SUCCESS; - // Find the frame where to read from - for(DWORD i = 0; i < hf->FrameCount; i++, pFrame++) + // Load the encoded frame to memory + if(FileStream_Read(pStream, &FileOffset, pbEncodedFrame, pFrame->EncodedSize)) { - // Does the read request fit into the current frame? - FrameBegin = pFrame->FrameFileOffset; - FrameEnd = FrameBegin + pFrame->FrameSize; - if(FrameBegin <= FilePointer && FilePointer < FrameEnd) - return pFrame; + if (bVerifyIntegrity) + { + if (!CascVerifyDataBlockHash(pbEncodedFrame, pFrame->EncodedSize, pFrame->FrameHash.Value)) + nError = ERROR_FILE_CORRUPT; + } + } + else + { + nError = GetLastError(); } - // Not found, sorry - return NULL; + return nError; } static int ProcessFileFrame( + TCascStorage * hs, LPBYTE pbOutBuffer, DWORD cbOutBuffer, LPBYTE pbInBuffer, DWORD cbInBuffer, DWORD dwFrameIndex) { - LPBYTE pbTempBuffer; - LPBYTE pbWorkBuffer; - DWORD cbTempBuffer = CASCLIB_MAX(cbInBuffer, cbOutBuffer); - DWORD cbWorkBuffer = cbOutBuffer + 1; + LPBYTE pbWorkBuffer = NULL; + DWORD cbOutBufferExpected = 0; + DWORD cbWorkBuffer = 0; DWORD dwStepCount = 0; bool bWorkComplete = false; int nError = ERROR_SUCCESS; - // Allocate the temporary buffer that will serve as output - pbWorkBuffer = pbTempBuffer = CASC_ALLOC(BYTE, cbTempBuffer); - if(pbWorkBuffer == NULL) - return ERROR_NOT_ENOUGH_MEMORY; - // Perform the loop - for(;;) + while(bWorkComplete == false) { - // Set the output buffer. - // Even operations: extract to temporary buffer - // Odd operations: extract to output buffer - pbWorkBuffer = (dwStepCount & 0x01) ? pbOutBuffer : pbTempBuffer; - cbWorkBuffer = (dwStepCount & 0x01) ? cbOutBuffer : cbTempBuffer; + // There should never be a 3rd step + assert(dwStepCount < 2); - // Perform the operation specific to the operation ID + // Perform the operation specific by the first byte switch(pbInBuffer[0]) { case 'E': // Encrypted files - nError = CascDecrypt(pbWorkBuffer, &cbWorkBuffer, pbInBuffer + 1, cbInBuffer - 1, dwFrameIndex); - bWorkComplete = (nError != ERROR_SUCCESS); + + // The work buffer should not have been allocated by any step + assert(pbWorkBuffer == NULL && cbWorkBuffer == 0); + + // Allocate temporary buffer to decrypt into + // Example storage: "2016 - WoW/23420", File: "4ee6bc9c6564227f1748abd0b088e950" + pbWorkBuffer = CASC_ALLOC(BYTE, cbInBuffer - 1); + cbWorkBuffer = cbInBuffer - 1; + if(pbWorkBuffer == NULL) + return ERROR_NOT_ENOUGH_MEMORY; + + // Decrypt the stream to the work buffer + nError = CascDecrypt(hs, pbWorkBuffer, &cbWorkBuffer, pbInBuffer + 1, cbInBuffer - 1, dwFrameIndex); + if(nError != ERROR_SUCCESS) + { + bWorkComplete = true; + break; + } + + // When encrypted, there is always one more step after this. + // Setup the work buffer as input buffer for the next operation + pbInBuffer = pbWorkBuffer; + cbInBuffer = cbWorkBuffer; break; case 'Z': // ZLIB compressed files - nError = CascDecompress(pbWorkBuffer, &cbWorkBuffer, pbInBuffer + 1, cbInBuffer - 1); + + // If we decompressed less than expected, we simply fill the rest with zeros + // Example: INSTALL file from the TACT CASC storage + cbOutBufferExpected = cbOutBuffer; + nError = CascDecompress(pbOutBuffer, &cbOutBuffer, pbInBuffer + 1, cbInBuffer - 1); + + // We exactly know what the output buffer size will be. + // If the uncompressed data is smaller, fill the rest with zeros + if(cbOutBuffer < cbOutBufferExpected) + memset(pbOutBuffer + cbOutBuffer, 0, (cbOutBufferExpected - cbOutBuffer)); bWorkComplete = true; break; case 'N': // Normal stored files - nError = CascDirectCopy(pbWorkBuffer, &cbWorkBuffer, pbInBuffer + 1, cbInBuffer - 1); + nError = CascDirectCopy(pbOutBuffer, &cbOutBuffer, pbInBuffer + 1, cbInBuffer - 1); bWorkComplete = true; break; @@ -320,48 +471,131 @@ static int ProcessFileFrame( break; } - // Are we done? - if(bWorkComplete) - break; - - // Set the input buffer to the work buffer - pbInBuffer = pbWorkBuffer; - cbInBuffer = cbWorkBuffer; + // Increment the step count dwStepCount++; } - // If the data are currently in the temporary buffer, - // we need to copy them to output buffer - if(nError == ERROR_SUCCESS && pbWorkBuffer != pbOutBuffer) + // Free the temporary buffer + CASC_FREE(pbWorkBuffer); + return nError; +} + +static bool GetFileFullInfo(TCascFile * hf, void * pvFileInfo, size_t cbFileInfo, size_t * pcbLengthNeeded) +{ + PCASC_FILE_FULL_INFO pFileInfo; + PCASC_CKEY_ENTRY pCKeyEntry = hf->pCKeyEntry; + TCascStorage * hs = hf->hs; + + // Verify whether we have enough space in the buffer + pFileInfo = (PCASC_FILE_FULL_INFO)ProbeOutputBuffer(pvFileInfo, cbFileInfo, sizeof(CASC_FILE_FULL_INFO), pcbLengthNeeded); + if(pFileInfo != NULL) { - if(cbWorkBuffer != cbOutBuffer) - nError = ERROR_INSUFFICIENT_BUFFER; - memcpy(pbOutBuffer, pbWorkBuffer, cbOutBuffer); + // Reset the entire structure + CopyMemory16(pFileInfo->CKey, pCKeyEntry->CKey); + CopyMemory16(pFileInfo->EKey, pCKeyEntry->EKey); + pFileInfo->FileDataId = CASC_INVALID_ID; + pFileInfo->LocaleFlags = CASC_INVALID_ID; + pFileInfo->ContentFlags = CASC_INVALID_ID; + + // Supply information not depending on root + CascStrPrintf(pFileInfo->DataFileName, _countof(pFileInfo->DataFileName), "data.%03u", hf->ArchiveIndex); + pFileInfo->StorageOffset = pCKeyEntry->StorageOffset; + pFileInfo->SegmentOffset = hf->ArchiveOffset; + pFileInfo->FileNameHash = 0; + pFileInfo->TagBitMask = pCKeyEntry->TagBitMask; + pFileInfo->SegmentIndex = hf->ArchiveIndex; + pFileInfo->ContentSize = hf->ContentSize; + pFileInfo->EncodedSize = hf->EncodedSize; + + // Supply the root-specific information + hs->pRootHandler->GetInfo(pCKeyEntry, pFileInfo); } - // Free the temporary buffer - CASC_FREE(pbTempBuffer); - return nError; + return (pFileInfo != NULL); } //----------------------------------------------------------------------------- // Public functions +bool WINAPI CascGetFileInfo(HANDLE hFile, CASC_FILE_INFO_CLASS InfoClass, void * pvFileInfo, size_t cbFileInfo, size_t * pcbLengthNeeded) +{ + TCascFile * hf; + LPBYTE pbOutputValue = NULL; + LPBYTE pbInfoValue = NULL; + size_t cbInfoValue = 0; + + // Validate the file handle + if((hf = TCascFile::IsValid(hFile)) == NULL) + { + SetLastError(ERROR_INVALID_HANDLE); + return false; + } + + // Differentiate between info classes + switch(InfoClass) + { + case CascFileContentKey: + + // Do we have content key at all? + if(hf->pCKeyEntry == NULL || (hf->pCKeyEntry->Flags & CASC_CE_HAS_CKEY) == 0) + { + SetLastError(ERROR_NOT_SUPPORTED); + return false; + } + + // Give the content key + pbInfoValue = hf->pCKeyEntry->CKey; + cbInfoValue = CASC_CKEY_SIZE; + break; + + case CascFileEncodedKey: + + // Do we have content key at all? + if(hf->pCKeyEntry == NULL || (hf->pCKeyEntry->Flags & CASC_CE_HAS_EKEY) == 0) + { + SetLastError(ERROR_NOT_SUPPORTED); + return false; + } + + // Give the content key + pbInfoValue = hf->pCKeyEntry->EKey; + cbInfoValue = CASC_CKEY_SIZE; + break; + + case CascFileFullInfo: + return GetFileFullInfo(hf, pvFileInfo, cbFileInfo, pcbLengthNeeded); + + default: + SetLastError(ERROR_INVALID_PARAMETER); + return false; + } + + // Sanity check + assert(pbInfoValue != NULL); + assert(cbInfoValue != 0); + + // Give the result + pbOutputValue = (LPBYTE)ProbeOutputBuffer(pvFileInfo, cbFileInfo, cbInfoValue, pcbLengthNeeded); + if(pbOutputValue != NULL) + memcpy(pbOutputValue, pbInfoValue, cbInfoValue); + return (pbOutputValue != NULL); +} + // // THE FILE SIZE PROBLEM // // There are members called "FileSize" in many CASC-related structure // For various files, these variables have different meaning. // -// Storage FileName FileSize FrameSum HdrArea IdxEntry EncEntry RootEntry -// ----------- -------- ---------- -------- -------- -------- -------- --------- -// HotS(29049) ENCODING 0x0024BA45 - 0x0024b98a 0x0024BA45 0x0024BA45 n/a n/a -// HotS(29049) ROOT 0x00193340 - 0x00193340 0x0010db65 0x0010db65 0x00193340 n/a -// HotS(29049) (other) 0x00001080 - 0x00001080 0x000008eb 0x000008eb 0x00001080 0x00001080 -// -// WoW(18888) ENCODING 0x030d487b - 0x030dee79 0x030d487b 0x030d487b n/a n/a -// WoW(18888) ROOT 0x016a9800 - n/a 0x0131313d 0x0131313d 0x016a9800 n/a -// WoW(18888) (other) 0x000007d0 - 0x000007d0 0x00000397 0x00000397 0x000007d0 n/a +// Storage FileName FileSize FrameSum HdrArea CKeyEntry EKeyEntry RootEntry +// ----------- -------- ---------- -------- -------- ---------- ---------- ---------- +// HotS(29049) ENCODING 0x0024BA45 - 0x0024b98a 0x0024BA45 n/a 0x0024BA45 n/a +// HotS(29049) ROOT 0x00193340 - 0x00193340 0x0010db65 0x00193340 0x0010db65 n/a +// HotS(29049) (other) 0x00001080 - 0x00001080 0x000008eb 0x00001080 0x000008eb 0x00001080 +// +// WoW(18888) ENCODING 0x030d487b - 0x030dee79 0x030d487b n/a 0x030d487b n/a +// WoW(18888) ROOT 0x016a9800 - n/a 0x0131313d 0x016a9800 0x0131313d n/a +// WoW(18888) (other) 0x000007d0 - 0x000007d0 0x00000397 0x000007d0 0x00000397 n/a // DWORD WINAPI CascGetFileSize(HANDLE hFile, PDWORD pdwFileSizeHigh) @@ -372,24 +606,32 @@ DWORD WINAPI CascGetFileSize(HANDLE hFile, PDWORD pdwFileSizeHigh) CASCLIB_UNUSED(pdwFileSizeHigh); // Validate the file handle - if((hf = IsValidFileHandle(hFile)) == NULL) + if((hf = TCascFile::IsValid(hFile)) == NULL) { SetLastError(ERROR_INVALID_HANDLE); return CASC_INVALID_SIZE; } - // Make sure that the file header area is loaded - nError = EnsureFrameHeadersLoaded(hf); - if(nError != ERROR_SUCCESS) + // Someone may have provided file content size. + // If yes, do not load the frames, as it's not necessary. + if(hf->ContentSize == CASC_INVALID_SIZE) { - SetLastError(nError); - return CASC_INVALID_SIZE; + // Make sure that the file header area is loaded + nError = EnsureFileFramesLoaded(hf); + if(nError != ERROR_SUCCESS) + { + SetLastError(nError); + return CASC_INVALID_SIZE; + } + + // The content size should be loaded from the frames + assert(hf->ContentSize != CASC_INVALID_SIZE); } // Give the file size to the caller if(pdwFileSizeHigh != NULL) *pdwFileSizeHigh = 0; - return hf->FileSize; + return hf->ContentSize; } DWORD WINAPI CascSetFilePointer(HANDLE hFile, LONG lFilePos, LONG * plFilePosHigh, DWORD dwMoveMethod) @@ -400,7 +642,7 @@ DWORD WINAPI CascSetFilePointer(HANDLE hFile, LONG lFilePos, LONG * plFilePosHig DWORD dwFilePosHi; // If the hFile is not a valid file handle, return an error. - hf = IsValidFileHandle(hFile); + hf = TCascFile::IsValid(hFile); if(hf == NULL) { SetLastError(ERROR_INVALID_HANDLE); @@ -419,7 +661,7 @@ DWORD WINAPI CascSetFilePointer(HANDLE hFile, LONG lFilePos, LONG * plFilePosHig break; case FILE_END: - FilePosition = hf->FileSize; + FilePosition = hf->ContentSize; break; default: @@ -458,16 +700,7 @@ DWORD WINAPI CascSetFilePointer(HANDLE hFile, LONG lFilePos, LONG * plFilePosHig bool WINAPI CascReadFile(HANDLE hFile, void * pvBuffer, DWORD dwBytesToRead, PDWORD pdwBytesRead) { - PCASC_FILE_FRAME pFrame = NULL; - ULONGLONG StreamSize; - ULONGLONG FileOffset; TCascFile * hf; - LPBYTE pbBuffer = (LPBYTE)pvBuffer; - DWORD dwStartPointer = 0; - DWORD dwFilePointer = 0; - DWORD dwEndPointer = 0; - DWORD dwFrameSize; - bool bReadResult; int nError = ERROR_SUCCESS; // The buffer must be valid @@ -478,7 +711,7 @@ bool WINAPI CascReadFile(HANDLE hFile, void * pvBuffer, DWORD dwBytesToRead, PDW } // Validate the file handle - if((hf = IsValidFileHandle(hFile)) == NULL) + if((hf = TCascFile::IsValid(hFile)) == NULL) { SetLastError(ERROR_INVALID_HANDLE); return false; @@ -487,134 +720,116 @@ bool WINAPI CascReadFile(HANDLE hFile, void * pvBuffer, DWORD dwBytesToRead, PDW // If the file frames are not loaded yet, do it now if(nError == ERROR_SUCCESS) { - nError = EnsureFrameHeadersLoaded(hf); + nError = EnsureFileFramesLoaded(hf); } // If the file position is at or beyond end of file, do nothing - if(nError == ERROR_SUCCESS && hf->FilePointer >= hf->FileSize) + if(nError == ERROR_SUCCESS) { - *pdwBytesRead = 0; - return true; + // Check the starting position + if(hf->FilePointer >= hf->ContentSize) + { + *pdwBytesRead = 0; + return true; + } + + // Check the ending position + if((hf->FilePointer + dwBytesToRead) > hf->ContentSize) + { + dwBytesToRead = hf->ContentSize - hf->FilePointer; + } } - // Find the file frame where to read from + // Allocate cache buffer for the entire file. This is the fastest approach + // (without reallocations). However, this may consume quite a lot of memory + // (Storage: "2016 - Starcraft II/45364", file: "3d815f40c0413701aa2bd214070d0062" + // needs 0x239a09b3 bytes of memory (~600 MB) if(nError == ERROR_SUCCESS) { - // Get the frame - pFrame = FindFileFrame(hf, hf->FilePointer); - if(pFrame == NULL || pFrame->CompressedSize < 1) - nError = ERROR_FILE_CORRUPT; + if(hf->pbFileCache == NULL) + { + // Allocate buffer + hf->pbFileCache = CASC_ALLOC(BYTE, hf->ContentSize); + hf->cbFileCache = hf->ContentSize; + if(hf->pbFileCache == NULL) + nError = ERROR_NOT_ENOUGH_MEMORY; + } } - // Perform the read + // Load all frames that are not loaded yet if(nError == ERROR_SUCCESS) { - // If not enough bytes in the file remaining, cut them - dwStartPointer = dwFilePointer = hf->FilePointer; - dwEndPointer = dwStartPointer + dwBytesToRead; - if(dwEndPointer > hf->FileSize) - dwEndPointer = hf->FileSize; - - // Perform block read from each file frame - while(dwFilePointer < dwEndPointer) + PCASC_FILE_FRAME pFrame = hf->pFrames; + DWORD StartFrameOffset = 0; + DWORD StartReadOffset = hf->FilePointer; + DWORD EndReadOffset = hf->FilePointer + dwBytesToRead; + + for(DWORD i = 0; (i < hf->FrameCount) && (nError == ERROR_SUCCESS); i++, pFrame++) { - LPBYTE pbFrameData = NULL; - DWORD dwFrameStart = pFrame->FrameFileOffset; - DWORD dwFrameEnd = pFrame->FrameFileOffset + pFrame->FrameSize; + LPBYTE pbDecodedFrame = hf->pbFileCache + StartFrameOffset; + LPBYTE pbEncodedFrame; + DWORD EndFrameOffset = StartFrameOffset + pFrame->ContentSize; - // Shall we populate the cache with a new data? - if(dwFrameStart != hf->CacheStart || hf->CacheEnd != dwFrameEnd) + // Does that frame belong to the range? + if(StartReadOffset < EndFrameOffset && EndReadOffset > StartFrameOffset) { - // Shall we reallocate the cache buffer? - if(pFrame->FrameSize > hf->cbFileCache) - { - if(hf->pbFileCache != NULL) - CASC_FREE(hf->pbFileCache); - - hf->pbFileCache = CASC_ALLOC(BYTE, pFrame->FrameSize); - hf->cbFileCache = pFrame->FrameSize; - } - - // We also need to allocate buffer for the raw data - pbFrameData = CASC_ALLOC(BYTE, pFrame->CompressedSize); - if(pbFrameData == NULL) - { - nError = ERROR_NOT_ENOUGH_MEMORY; - break; - } - - // Load the raw file data to memory - FileOffset = pFrame->FrameArchiveOffset; - bReadResult = FileStream_Read(hf->pStream, &FileOffset, pbFrameData, pFrame->CompressedSize); - - // Note: The raw file data size could be less than expected - // Happened in WoW build 19342 with the ROOT file. MD5 in the frame header - // is zeroed, which means it should not be checked - // Frame File: data.029 - // Frame Offs: 0x013ED9F0 size 0x01325B32 - // Frame End: 0x02713522 - // File Size: 0x027134FC - if(bReadResult == false && GetLastError() == ERROR_HANDLE_EOF && !IsValidMD5(pFrame->md5)) + // Is the frame already loaded? + if (pFrame->FileOffset == CASC_INVALID_POS) { - // Get the size of the remaining file - FileStream_GetSize(hf->pStream, &StreamSize); - dwFrameSize = (DWORD)(StreamSize - FileOffset); - - // If the frame offset is before EOF and frame end is beyond EOF, correct it - if(FileOffset < StreamSize && dwFrameSize < pFrame->CompressedSize) + // Allocate space for the encoded frame + pbEncodedFrame = CASC_ALLOC(BYTE, pFrame->EncodedSize); + if (pbEncodedFrame != NULL) { - memset(pbFrameData + dwFrameSize, 0, (pFrame->CompressedSize - dwFrameSize)); - bReadResult = true; + // Load the encoded frame data + nError = LoadEncodedFrame(hf->pStream, pFrame, pbEncodedFrame, hf->bVerifyIntegrity); + if (nError == ERROR_SUCCESS) + { + // Decode the frame + nError = ProcessFileFrame(hf->hs, + pbDecodedFrame, + pFrame->ContentSize, + pbEncodedFrame, + pFrame->EncodedSize, + (DWORD)(pFrame - hf->pFrames)); + if (nError == ERROR_SUCCESS) + { + // Mark the frame as loaded + pFrame->FileOffset = StartFrameOffset; + } + } + + // Free the frame buffer + CASC_FREE(pbEncodedFrame); } - } - - // If the read result failed, we cannot finish reading it - if(bReadResult && VerifyDataBlockHash(pbFrameData, pFrame->CompressedSize, pFrame->md5)) - { - // Convert the source frame to the file cache - nError = ProcessFileFrame(hf->pbFileCache, - pFrame->FrameSize, - pbFrameData, - pFrame->CompressedSize, - (DWORD)(pFrame - hf->pFrames)); - if(nError == ERROR_SUCCESS) + else { - // Set the start and end of the cache - hf->CacheStart = dwFrameStart; - hf->CacheEnd = dwFrameEnd; + nError = ERROR_NOT_ENOUGH_MEMORY; } } - else - { - nError = ERROR_FILE_CORRUPT; - } - - // Free the raw frame data - CASC_FREE(pbFrameData); } - // Copy the decompressed data - if(dwFrameEnd > dwEndPointer) - dwFrameEnd = dwEndPointer; - memcpy(pbBuffer, hf->pbFileCache + (dwFilePointer - dwFrameStart), (dwFrameEnd - dwFilePointer)); - pbBuffer += (dwFrameEnd - dwFilePointer); - - // Move pointers - dwFilePointer = dwFrameEnd; - pFrame++; + // If the frame start is past the read offset, stop the loop + if ((StartFrameOffset + pFrame->ContentSize) >= EndReadOffset) + break; + StartFrameOffset += pFrame->ContentSize; } } - // Update the file position + // Now all frames have been loaded into the cache; copy the entire block to the output buffer if(nError == ERROR_SUCCESS) { + // Copy the entire data + memcpy(pvBuffer, hf->pbFileCache + hf->FilePointer, dwBytesToRead); + hf->FilePointer += dwBytesToRead; + + // Give the number of bytes read if(pdwBytesRead != NULL) - *pdwBytesRead = (dwFilePointer - dwStartPointer); - hf->FilePointer = dwFilePointer; + *pdwBytesRead = dwBytesToRead; + return true; } - - if(nError != ERROR_SUCCESS) + else + { SetLastError(nError); - return (nError == ERROR_SUCCESS); + return false; + } } - -- cgit v1.2.3