path: root/dep/CascLib/src/CascCommon.h

/*****************************************************************************/
/* CascCommon.h                           Copyright (c) Ladislav Zezula 2014 */
/*---------------------------------------------------------------------------*/
/* Common functions for CascLib                                              */
/*---------------------------------------------------------------------------*/
/*   Date    Ver   Who  Comment                                              */
/* --------  ----  ---  -------                                              */
/* 29.04.14  1.00  Lad  The first version of CascCommon.h                    */
/*****************************************************************************/

#ifndef __CASCCOMMON_H__
#define __CASCCOMMON_H__

//-----------------------------------------------------------------------------
// Compression support

// Include functions from zlib
#ifndef __SYS_ZLIB
  #include "zlib/zlib.h"
#else
  #include <zlib.h>
#endif

#include "CascPort.h"
#include "common/Common.h"
#include "common/Array.h"
#include "common/Map.h"
#include "common/FileTree.h"
#include "common/FileStream.h"
#include "common/Directory.h"
#include "common/ListFile.h"
#include "common/Csv.h"
#include "common/RootHandler.h"

// Headers from Alexander Peslyak's MD5 implementation
#include "md5/md5.h"

// For HashStringJenkins
#include "jenkins/lookup.h"

// On-disk CASC structures
#include "CascStructs.h"

//-----------------------------------------------------------------------------
// CascLib private defines

#ifdef _DEBUG
#define BREAK_ON_XKEY3(CKey, v0, v1, v2) if(CKey[0] == v0 && CKey[1] == v1 && CKey[2] == v2) { __debugbreak(); }
#define BREAKIF(condition)               if(condition)  { __debugbreak(); }
#else
#define BREAK_ON_XKEY3(CKey, v0, v1, v2) { /* NOTHING */ }
#define BREAKIF(condition)               { /* NOTHING */ }
#endif

//-----------------------------------------------------------------------------
// In-memory structures

typedef enum _CBLD_TYPE
{
    CascBuildNone = 0,                              // No build type found
    CascBuildInfo,                                  // .build.info
    CascBuildDb,                                    // .build.db (older storages)
    CascVersionsDb                                  // versions (downloaded online)
} CBLD_TYPE, *PCBLD_TYPE;

// Tag file entry, loaded from the DOWNLOAD file
typedef struct _CASC_TAG_ENTRY
{
    USHORT HashType;                                // Hash type
    char TagName[1];                                // Tag name. Variable length.
} CASC_TAG_ENTRY, *PCASC_TAG_ENTRY;

// Normalized header of the index files.
// Both version 1 and version 2 are converted to this structure
typedef struct _CASC_INDEX_HEADER
{
    USHORT IndexVersion;                            // 5 for index v 1.0, 7 for index version 2.0
    BYTE   BucketIndex;                             // Should be the same as the first byte of the hex filename. 
    BYTE   StorageOffsetLength;                     // Length, in bytes, of the StorageOffset field in the EKey entry
    BYTE   EncodedSizeLength;                       // Length, in bytes, of the EncodedSize in the EKey entry
    BYTE   EKeyLength;                              // Length, in bytes, of the (trimmed) EKey in the EKey entry
    BYTE   FileOffsetBits;                          // Number of bits of the archive file offset in StorageOffset field. Rest is data segment index
    BYTE   Alignment;
    ULONGLONG SegmentSize;                          // Size of one data segment (aka data.### file)
    size_t HeaderLength;                            // Length of the on-disk header structure, in bytes
    size_t HeaderPadding;                           // Length of padding after the header
    size_t EntryLength;                             // Length of the on-disk EKey entry structure, in bytes
    size_t EKeyCount;                               // Number of EKey entries. Only supplied for index files version 1.

} CASC_INDEX_HEADER, *PCASC_INDEX_HEADER;

// Normalized footer of the archive index files (md5.index)
typedef struct _CASC_ARCINDEX_FOOTER
{
    BYTE   TocHash[MD5_HASH_SIZE];                  // Hash of the structure
    BYTE   FooterHash[MD5_HASH_SIZE];               // MD5 hash of the footer, possibly shortened
    BYTE   Version;                                 // Version of the index footer
    BYTE   OffsetBytes;                             // Length, in bytes, of the file offset field
    BYTE   SizeBytes;                               // Length, in bytes, of the file size field
    BYTE   EKeyBytes;                               // Length, in bytes, of the EKey field
    BYTE   FooterHashBytes;                         // Length, in bytes, of the hash and checksum
    BYTE   Alignment[3];
    size_t ElementCount;                            // total number of elements in the index file
    size_t PageLength;                              // Length, in bytes, of the index page
    size_t ItemLength;                              // Length, in bytes, of the single item
    size_t FooterLength;                            // Length, in bytes, of the index footer structure

} CASC_ARCINDEX_FOOTER, *PCASC_ARCINDEX_FOOTER;

// Normalized structure for archive index entry
typedef struct _CASC_ARCINDEX_ENTRY
{
    BYTE IndexHash[MD5_HASH_SIZE];
    BYTE EKey[MD5_HASH_SIZE];
    DWORD ArchiveOffset;
    DWORD EncodedSize;
} CASC_ARCINDEX_ENTRY, *PCASC_ARCINDEX_ENTRY;

// Normalized header of the ENCODING file
typedef struct _CASC_ENCODING_HEADER
{
    USHORT Magic;                                   // FILE_MAGIC_ENCODING ('EN')
    USHORT Version;                                 // Expected to be 1 by CascLib
    BYTE   CKeyLength;                              // The content key length in ENCODING file. Usually 0x10
    BYTE   EKeyLength;                              // The encoded key length in ENCODING file. Usually 0x10
    DWORD  CKeyPageSize;                            // Size of the CKey page in bytes
    DWORD  EKeyPageSize;                            // Size of the CKey page in bytes
    DWORD  CKeyPageCount;                           // Number of CKey pages in the page table
    DWORD  EKeyPageCount;                           // Number of EKey pages in the page table
    DWORD  ESpecBlockSize;                          // Size of the ESpec string block, in bytes
} CASC_ENCODING_HEADER, *PCASC_ENCODING_HEADER;

typedef struct _CASC_DOWNLOAD_HEADER
{
    USHORT Magic;                                   // FILE_MAGIC_DOWNLOAD ('DL')
    USHORT Version;                                 // Version
    USHORT EKeyLength;                              // Length of the EKey
    USHORT EntryHasChecksum;                        // If nonzero, then the entry has checksum
    DWORD  EntryCount;
    DWORD  TagCount;
    USHORT FlagByteSize;
    USHORT BasePriority;

    size_t HeaderLength;                            // Length of the on-disk header, in bytes
    size_t EntryLength;                             // Length of the on-disk entry, in bytes

} CASC_DOWNLOAD_HEADER, *PCASC_DOWNLOAD_HEADER;

typedef struct _CASC_DOWNLOAD_ENTRY
{
    BYTE EKey[MD5_HASH_SIZE];
    ULONGLONG EncodedSize;
    DWORD Checksum;
    DWORD Flags;
    BYTE Priority;
} CASC_DOWNLOAD_ENTRY, *PCASC_DOWNLOAD_ENTRY;

// Capturable tag structure for loading from DOWNLOAD manifest
typedef struct _CASC_TAG_ENTRY1
{
    const char * szTagName;                         // Tag name
    LPBYTE Bitmap;                                  // Bitmap
    size_t BitmapLength;                            // Length of the bitmap, in bytes
    size_t NameLength;                              // Length of the tag name, in bytes, not including '\0'
    size_t TagLength;                               // Length of the on-disk tag, in bytes
    DWORD TagValue;                                 // Tag value
} CASC_TAG_ENTRY1, *PCASC_TAG_ENTRY1;

// Tag structure for storing in arrays
typedef struct _CASC_TAG_ENTRY2
{
    size_t NameLength;                              // Length of the on-disk tag, in bytes
    DWORD TagValue;                                 // Tag value
    char szTagName[0x08];                           // Tag string. This member can be longer than declared. Aligned to 8 bytes.
} CASC_TAG_ENTRY2, *PCASC_TAG_ENTRY2;

typedef struct _CASC_INSTALL_HEADER
{
    USHORT Magic;                                   // FILE_MAGIC_DOWNLOAD ('DL')
    BYTE   Version;                                 // Version
    BYTE   EKeyLength;                              // Length of the EKey
    DWORD  EntryCount;
    DWORD  TagCount;

    size_t HeaderLength;                            // Length of the on-disk header, in bytes
} CASC_INSTALL_HEADER, *PCASC_INSTALL_HEADER;

typedef struct _CASC_FILE_FRAME
{
    CONTENT_KEY FrameHash;                          // MD5 hash of the file frame
    DWORD DataFileOffset;                           // Offset in the data file (data.###)
    DWORD FileOffset;                               // File offset of this frame
    DWORD EncodedSize;                              // Encoded size of the frame
    DWORD ContentSize;                              // Content size of the frame
} CASC_FILE_FRAME, *PCASC_FILE_FRAME;

//-----------------------------------------------------------------------------
// Structures for CASC storage and CASC file

struct TCascStorage
{
    TCascStorage();
    ~TCascStorage();
    TCascStorage * AddRef();
    TCascStorage * Release();

    static TCascStorage * IsValid(HANDLE hStorage);

    PFNPROGRESSCALLBACK PfnCallback;                // Callback to be called during CascOpen(Online)Storage
    void * PtrCallbackParam;

    const TCHAR * szIndexFormat;                    // Format of the index file name
    const char * szClassName;                       // "TCascStorage"
    const char * szProductName;                     // String representation of the product name
    TCHAR * szRootPath;                             // Path where the build file is
    TCHAR * szDataPath;                             // This is the directory where data files are
    TCHAR * szIndexPath;                            // This is the directory where index files are
    TCHAR * szBuildFile;                            // Build file name (.build.info or .build.db)
    TCHAR * szCdnServers;                           // Multi-SZ list of CDN servers
    TCHAR * szCdnPath;                              // Remote CDN sub path for the product
    TCHAR * szCodeName;                             // Product code name. Only for online storages
    char * szRegion;                                // Product region. Only when "versions" is used as storage root file
    CASC_PRODUCT Product;                           // Product enum value (see CASC_PRODUCT)
    DWORD dwBuildNumber;                            // Product build number
    DWORD dwRefCount;                               // Number of references
    DWORD dwDefaultLocale;                          // Default locale, read from ".build.info"
    DWORD dwFeatures;                               // List of CASC features. See CASC_FEATURE_XXX
    bool bAllowOrphans;                             // If TRUE, then orphaned items are allowed

    CBLD_TYPE BuildFileType;                        // Type of the build file

    QUERY_KEY CdnConfigKey;                         // Currently selected CDN config file. Points to "config\%02X\%02X\%s
    QUERY_KEY CdnBuildKey;                          // Currently selected CDN build file. Points to "config\%02X\%02X\%s

    QUERY_KEY ArchiveGroup;                         // Key array of the "archive-group"
    QUERY_KEY ArchivesKey;                          // Key array of the "archives"
    QUERY_KEY PatchArchivesKey;                     // Key array of the "patch-archives"
    QUERY_KEY PatchArchivesGroup;                   // Key array of the "patch-archive-group"
    QUERY_KEY BuildFiles;                           // List of supported build files

    TFileStream * DataFiles[CASC_MAX_DATA_FILES];   // Array of open data files

    CASC_CKEY_ENTRY EncodingCKey;                   // Information about ENCODING file
    CASC_CKEY_ENTRY DownloadCKey;                   // Information about DOWNLOAD file
    CASC_CKEY_ENTRY InstallCKey;                    // Information about INSTALL file
    CASC_CKEY_ENTRY PatchFile;                      // Information about PATCH file
    CASC_CKEY_ENTRY RootFile;                       // Information about ROOT file
    CASC_CKEY_ENTRY SizeFile;                       // Information about SIZE file
    CASC_CKEY_ENTRY VfsRoot;                        // The main VFS root file
    CASC_ARRAY VfsRootList;                         // List of CASC_EKEY_ENTRY for each TVFS sub-root

    TRootHandler * pRootHandler;                    // Common handler for various ROOT file formats
    CASC_ARRAY ArcIndexArray;                       // Array of CASC_ARCINDEX_ENTRY, loaded from archive indexes
    CASC_ARRAY IndexArray;                          // Array of CASC_CKEY_ENTRY loaded from local index files
    CASC_ARRAY CKeyArray;                           // Array of CASC_CKEY_ENTRY, one entry for each physical file
    CASC_ARRAY TagsArray;                           // Array of CASC_DOWNLOAD_TAG2
    CASC_MAP ArcIndexMap;                           // Map of EKey -> CASC_ARCINDEX_ENTRY
    CASC_MAP CKeyMap;                               // Map of CKey -> CASC_CKEY_ENTRY
    CASC_MAP EKeyMap;                               // Map of EKey -> CASC_EKEY_ENTRY
    size_t LocalFiles;                              // Number of files that are present locally
    size_t TotalFiles;                              // Total number of files in the storage, some may not be present locally
    size_t EKeyEntries;                             // Number of CKeyEntry-ies loaded from text build file
    size_t OrphanItems;                             // Number of EKey entries in indexes that do not have their counterpart in ENCODING
    size_t SkippedItems;                            // Number of EKey entries in indexes that were ignored due to insufficient capacity of CKeyArray
    size_t EKeyLength;                              // EKey length from the index files
    DWORD FileOffsetBits;                           // Nimber of bits in the storage offset which mean data segent offset

    CASC_ARRAY ExtraKeysList;                       // List additional encryption keys
    CASC_MAP   EncryptionKeys;                      // Map of encryption keys

};

struct TCascFile
{
    TCascFile(TCascStorage * ahs, PCASC_CKEY_ENTRY apCKeyEntry)
    {
        ULONGLONG StorageOffset;
        ULONGLONG FileOffsMask;

        // Reference the storage handle
        hs = ahs->AddRef();
        szClassName = "TCascFile";

        // Init class variables
        pCKeyEntry = apCKeyEntry;
        pFrames = NULL;
        pbFileCache = NULL;
        cbFileCache = 0;

        // Supply the sizes
        StorageOffset = pCKeyEntry->StorageOffset;
        FileOffsMask = ((ULONGLONG)1 << hs->FileOffsetBits) - 1;
        ArchiveIndex = (DWORD)(StorageOffset >> hs->FileOffsetBits);
        ArchiveOffset = (DWORD)(StorageOffset & FileOffsMask);
        EncodedSize = pCKeyEntry->EncodedSize;
        ContentSize = pCKeyEntry->ContentSize;
        FilePointer = 0;
        FrameCount = 0;
        bVerifyIntegrity = false;
        bDownloadFileIf = false;
        bLocalFileStream = false;
    }

    ~TCascFile()
    {
        // Close (dereference) the archive handle
        hs = hs->Release();
        szClassName = NULL;

        // Free the file cache and frame array
        CASC_FREE(pbFileCache);
        CASC_FREE(pFrames);

        // If we are supposed to close the file stream, do it
        if (pStream && bLocalFileStream)
            FileStream_Close(pStream);
    }

    static TCascFile * IsValid(HANDLE hFile)
    {
        TCascFile * hf = (TCascFile *)hFile;

        return (hf != INVALID_HANDLE_VALUE &&
                hf != NULL &&
                hf->hs != NULL &&
                hf->szClassName != NULL &&
                hf->pCKeyEntry != NULL) ? hf : NULL;
    }

    TCascStorage * hs;                              // Pointer to storage structure
    TFileStream * pStream;                          // An open data stream
    const char * szClassName;                       // "TCascFile"

    PCASC_CKEY_ENTRY pCKeyEntry;
    PCASC_FILE_FRAME pFrames;                       // Array of file frames
    DWORD ArchiveIndex;                             // Index of the archive (data.###)
    DWORD ArchiveOffset;                            // Offset in the archive (data.###)
    DWORD FilePointer;                              // Current file pointer
    DWORD EncodedSize;                              // Encoded size. This is the size of encoded header, all file frame headers and all file frames
    DWORD ContentSize;                              // Content size. This is the size of the file content, aka the file size
    DWORD FrameCount;                               // Number of the file frames
    DWORD bVerifyIntegrity:1;                       // If true, then the data are validated more strictly when read
    DWORD bDownloadFileIf:1;                        // If true, then the data will be downloaded from the online storage if missing
    DWORD bLocalFileStream:1;                       // If true, then the file stream is a local file
    DWORD bOvercomeEncrypted:1;                     // If true, then CascReadFile will fill the part that is encrypted (and key was not found) with zeros 

    LPBYTE pbFileCache;                             // Pointer to file cache
    DWORD cbFileCache;                              // Size of the file cache

};

struct TCascSearch
{
    TCascSearch(TCascStorage * ahs, LPCTSTR aszListFile, const char * aszMask)
    {
        // Init the class
        szClassName = "TCascSearch";
        hs = ahs->AddRef();

        // Init provider-specific data
        pCache = NULL;
        nFileIndex = 0;
        nSearchState = 0;
        bListFileUsed = false;

        // Allocate mask
        szListFile = CascNewStr(aszListFile);
        szMask = CascNewStr((aszMask != NULL) ? aszMask : "*");
    }

    ~TCascSearch()
    {
        // Dereference the CASC storage
        hs = hs->Release();
        szClassName = NULL;

        // Free the rest of the members
        CASC_FREE(szMask);
        CASC_FREE(szListFile);
        CASC_FREE(pCache);
    }

    static TCascSearch * IsValid(HANDLE hFind)
    {
        TCascSearch * pSearch = (TCascSearch *)hFind;

        return (hFind != INVALID_HANDLE_VALUE &&
                hFind != NULL &&
                pSearch->szClassName != NULL &&
               !strcmp(pSearch->szClassName, "TCascSearch") &&
                pSearch->szMask != NULL) ? pSearch : NULL;
    }

    TCascStorage * hs;                              // Pointer to the storage handle
    const char * szClassName;                       // Contains "TCascSearch"
    TCHAR * szListFile;                             // Name of the listfile
    void * pCache;                                  // Listfile cache
    char * szMask;                                  // Search mask

    // Provider-specific data
    size_t nFileIndex;                              // Root-specific search context
    DWORD nSearchState:8;                           // The current search state (0 = listfile, 1 = nameless, 2 = done)
    DWORD bListFileUsed:1;                          // TRUE: The listfile has already been loaded
};

//-----------------------------------------------------------------------------
// Common functions (CascCommon.cpp)

bool OpenFileByCKeyEntry(TCascStorage * hs, PCASC_CKEY_ENTRY pCKeyEntry, DWORD dwOpenFlags, HANDLE * PtrFileHandle);
LPBYTE LoadInternalFileToMemory(TCascStorage * hs, PCASC_CKEY_ENTRY pCKeyEntry, DWORD * pcbFileData);
LPBYTE LoadFileToMemory(const TCHAR * szFileName, DWORD * pcbFileData);
void FreeCascBlob(PQUERY_KEY pQueryKey);

//-----------------------------------------------------------------------------
// Text file parsing (CascFiles.cpp)

int DownloadFileFromCDN(TCascStorage * hs, const TCHAR * szSubDir, LPBYTE pbEKey, const TCHAR * szExtension, TCHAR * szOutLocalPath, size_t cchOutLocalPath);
int CheckGameDirectory(TCascStorage * hs, TCHAR * szDirectory);
int LoadCdnsInfo(TCascStorage * hs);
int LoadBuildInfo(TCascStorage * hs);
int LoadCdnConfigFile(TCascStorage * hs);
int LoadCdnBuildFile(TCascStorage * hs);

//-----------------------------------------------------------------------------
// Internal file functions

void * ProbeOutputBuffer(void * pvBuffer, size_t cbLength, size_t cbMinLength, size_t * pcbLengthNeeded);

PCASC_CKEY_ENTRY FindCKeyEntry_CKey(TCascStorage * hs, LPBYTE pbCKey, PDWORD PtrIndex = NULL);
PCASC_CKEY_ENTRY FindCKeyEntry_EKey(TCascStorage * hs, LPBYTE pbEKey, PDWORD PtrIndex = NULL);

size_t GetTagBitmapLength(LPBYTE pbFilePtr, LPBYTE pbFileEnd, DWORD EntryCount);

int CascDecompress(LPBYTE pvOutBuffer, PDWORD pcbOutBuffer, LPBYTE pvInBuffer, DWORD cbInBuffer);
int CascDirectCopy(LPBYTE pbOutBuffer, PDWORD pcbOutBuffer, LPBYTE pbInBuffer, DWORD cbInBuffer);

int CascLoadEncryptionKeys(TCascStorage * hs);
int CascDecrypt(TCascStorage * hs, LPBYTE pbOutBuffer, PDWORD pcbOutBuffer, LPBYTE pbInBuffer, DWORD cbInBuffer, DWORD dwFrameIndex);

//-----------------------------------------------------------------------------
// Support for index files

int LoadIndexFiles(TCascStorage * hs);

//-----------------------------------------------------------------------------
// Support for ROOT file

int RootHandler_CreateMNDX(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateTVFS(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateDiablo3(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateWoW(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile, DWORD dwLocaleMask);
int RootHandler_CreateOverwatch(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateStarcraft1(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateInstall(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);

//-----------------------------------------------------------------------------
// Dumpers (CascDumpData.cpp)

#ifdef _DEBUG
void CascDumpFile(HANDLE hFile, const char * szDumpFile = NULL);
void CascDumpStorage(HANDLE hStorage, const char * szDumpFile = NULL);
#endif

#endif // __CASCCOMMON_H__