<commit_before>﻿#include <algorithm>
#include "OfficeFileFormatChecker.h"
#include "DocxFormat/Source/SystemUtility/File.h"
#include "../ASCOfficeUtils/ASCOfficeUtilsLib/OfficeUtils.h"

#define MIN_SIZE_BUFFER 4096
#define MAX_SIZE_BUFFER 102400

bool COfficeFileFormatChecker::isRtfFormatFile(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (5 <= dwBytes) && (  '{' == pBuffer[0] && '\\' == pBuffer[1] && 'r' == pBuffer[2] && 't' == pBuffer[3] && 'f' == pBuffer[4] ))
		return true;

	return false;
}
bool COfficeFileFormatChecker::isHtmlFormatFile(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

    bool tagOpen = false;
    //Html File is XML-file with rootElement - html
    for (int i = 0; i < dwBytes - 4 && i < 100; i++)
    {
        if (0x3C == pBuffer[i])
            tagOpen = true;
        else if (0x3E == pBuffer[i])
            tagOpen = false;
        else if (tagOpen && (0x48 == pBuffer[i] || 0x68 == pBuffer[i]) && (0x54 == pBuffer[i + 1] || 0x74 == pBuffer[i + 1])
                         && (0x4d == pBuffer[i + 2] || 0x6d == pBuffer[i + 2]) && (0x4c == pBuffer[i + 3] || 0x6c == pBuffer[i + 3]))
            return true;
    }
    return false;
}

bool COfficeFileFormatChecker::isBinaryDoctFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (4 <= dwBytes) && (  'D' == pBuffer[0] && 'O' == pBuffer[1] && 'C' == pBuffer[2] && 'Y' == pBuffer[3] ))
		return true;

	 return false;
}
bool COfficeFileFormatChecker::isBinaryXlstFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (4 <= dwBytes) && (  'X' == pBuffer[0] && 'L' == pBuffer[1] && 'S' == pBuffer[2] && 'Y' == pBuffer[3] ))
		return true;

	 return false;
}
bool COfficeFileFormatChecker::isBinaryPpttFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (4 <= dwBytes) && (  'P' == pBuffer[0] && 'P' == pBuffer[1] && 'T' == pBuffer[2] && 'Y' == pBuffer[3] ))
		return true;

	return false;
}
bool COfficeFileFormatChecker::isPdfFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

    int nTempBufferSize = dwBytes < 20 ? dwBytes : 20;
    char* pTempBuffer = new char[nTempBufferSize];

    memcpy ( pTempBuffer, pBuffer, nTempBufferSize );
    pTempBuffer[nTempBufferSize - 1] = '\0';

    char* pFirst = strstr( (char*)pTempBuffer, "%PDF-" );
    if( NULL != pFirst )
        return true;

	return false;
}
bool COfficeFileFormatChecker::isDocFormatFile	(POLE::Storage * storage)
{
	if (storage == false) return false;

	POLE::Stream stream(storage, "WordDocument");	
	
	unsigned char buffer[10];
	if (stream.read(buffer,10) >0)
		return true;

	return false;
}
bool COfficeFileFormatChecker::isXlsFormatFile	(POLE::Storage * storage)
{
	if (storage == false) return false;

	POLE::Stream stream(storage, "Workbook");
	
	unsigned char buffer[10];
	if (stream.read(buffer,10) >0)
		return true;

	return false;
}
bool COfficeFileFormatChecker::isPptFormatFile	(POLE::Storage * storage)
{
	if (storage == false) return false;

	POLE::Stream stream(storage, "PowerPoint Document");	

	unsigned char buffer[10];
	if (stream.read(buffer,10) >0)
		return true;

	return false;
}
bool COfficeFileFormatChecker::isMS_OFFCRYPTOFormatFile	(POLE::Storage * storage)
{
    if (storage == false) return false;

    std::list<std::string> entries = storage->entries("DataSpaces");
    if (entries.size() > 0)
        return true;

    return false;
}
bool COfficeFileFormatChecker::isOfficeFile(const std::wstring & fileName)
{
    //приоритет как оказывается важен
    //Metamorphic Manual for windows 28415.doc
	POLE::Storage storage(fileName.c_str());
    if (storage.open())
    {
        if ( isDocFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_DOC;
            return true;
        }
        else if ( isXlsFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLS;
            return true;
        }
        else if ( isPptFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_PRESENTATION_PPT;
            return true;
        }
        else if ( isMS_OFFCRYPTOFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_OTHER_MS_OFFCRYPTO;
            return true;
        }
    }

    COfficeUtils OfficeUtils(NULL);
    if (OfficeUtils.IsArchive(fileName) == S_OK)
	{
             if ( isOOXFormatFile(fileName) )           return true;
        else if ( isOpenOfficeFormatFile(fileName) )    return true;
        else if ( isOnlyOfficeFormatFile(fileName) )    return true;
        else if ( isXpsFile(fileName) )                 return true;
	}


    // others
    {
		CFile file;
        if (file.OpenFile(std_string2string(fileName)) != S_OK ) return _T("");
		
		unsigned char* buffer = new unsigned char[4096]; //enaf !!
		if (!buffer){file.CloseFile();return _T("");}

		file.ReadFile(buffer,MIN_SIZE_BUFFER);
		int sizeRead = file.GetPosition();
		file.CloseFile();

		if ( isRtfFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_RTF;
		}
		else if ( isBinaryDoctFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_CANVAS_WORD;
		}
		else if ( isBinaryXlstFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_CANVAS_SPREADSHEET;
		}
		else if ( isBinaryPpttFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_CANVAS_PRESENTATION;
		}
        else if (isHtmlFormatFile(buffer,sizeRead) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_HTML;
        }
        else if (isFB2FormatFile(buffer,sizeRead) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_HTML;
        }
        else if (isPdfFormatFile(buffer,sizeRead) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF;
        }
        else if (isDjvuFormatFile(buffer,sizeRead) )
        {<commit_msg>обрабатываем mht как html<commit_after>﻿#include <algorithm>
#include "OfficeFileFormatChecker.h"
#include "DocxFormat/Source/SystemUtility/File.h"
#include "../ASCOfficeUtils/ASCOfficeUtilsLib/OfficeUtils.h"

#define MIN_SIZE_BUFFER 4096
#define MAX_SIZE_BUFFER 102400

bool COfficeFileFormatChecker::isRtfFormatFile(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (5 <= dwBytes) && (  '{' == pBuffer[0] && '\\' == pBuffer[1] && 'r' == pBuffer[2] && 't' == pBuffer[3] && 'f' == pBuffer[4] ))
		return true;

	return false;
}
bool COfficeFileFormatChecker::isHtmlFormatFile(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

    bool tagOpen = false;
    //Html File is XML-file with rootElement - html
    for (int i = 0; i < dwBytes - 4 && i < 100; i++)
    {
        if (0x3C == pBuffer[i])
            tagOpen = true;
        else if (0x3E == pBuffer[i])
            tagOpen = false;
        else if (tagOpen && (0x48 == pBuffer[i] || 0x68 == pBuffer[i]) && (0x54 == pBuffer[i + 1] || 0x74 == pBuffer[i + 1])
                         && (0x4d == pBuffer[i + 2] || 0x6d == pBuffer[i + 2]) && (0x4c == pBuffer[i + 3] || 0x6c == pBuffer[i + 3]))
            return true;
    }
    return false;
}

bool COfficeFileFormatChecker::isBinaryDoctFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (4 <= dwBytes) && (  'D' == pBuffer[0] && 'O' == pBuffer[1] && 'C' == pBuffer[2] && 'Y' == pBuffer[3] ))
		return true;

	 return false;
}
bool COfficeFileFormatChecker::isBinaryXlstFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (4 <= dwBytes) && (  'X' == pBuffer[0] && 'L' == pBuffer[1] && 'S' == pBuffer[2] && 'Y' == pBuffer[3] ))
		return true;

	 return false;
}
bool COfficeFileFormatChecker::isBinaryPpttFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

	if ( (4 <= dwBytes) && (  'P' == pBuffer[0] && 'P' == pBuffer[1] && 'T' == pBuffer[2] && 'Y' == pBuffer[3] ))
		return true;

	return false;
}
bool COfficeFileFormatChecker::isPdfFormatFile	(unsigned char* pBuffer,int dwBytes)
{
	if (pBuffer == NULL) return false;

    int nTempBufferSize = dwBytes < 20 ? dwBytes : 20;
    char* pTempBuffer = new char[nTempBufferSize];

    memcpy ( pTempBuffer, pBuffer, nTempBufferSize );
    pTempBuffer[nTempBufferSize - 1] = '\0';

    char* pFirst = strstr( (char*)pTempBuffer, "%PDF-" );
    if( NULL != pFirst )
        return true;

	return false;
}
bool COfficeFileFormatChecker::isDocFormatFile	(POLE::Storage * storage)
{
	if (storage == false) return false;

	POLE::Stream stream(storage, "WordDocument");	
	
	unsigned char buffer[10];
	if (stream.read(buffer,10) >0)
		return true;

	return false;
}
bool COfficeFileFormatChecker::isXlsFormatFile	(POLE::Storage * storage)
{
	if (storage == false) return false;

	POLE::Stream stream(storage, "Workbook");
	
	unsigned char buffer[10];
	if (stream.read(buffer,10) >0)
		return true;

	return false;
}
bool COfficeFileFormatChecker::isPptFormatFile	(POLE::Storage * storage)
{
	if (storage == false) return false;

	POLE::Stream stream(storage, "PowerPoint Document");	

	unsigned char buffer[10];
	if (stream.read(buffer,10) >0)
		return true;

	return false;
}
bool COfficeFileFormatChecker::isMS_OFFCRYPTOFormatFile	(POLE::Storage * storage)
{
    if (storage == false) return false;

    std::list<std::string> entries = storage->entries("DataSpaces");
    if (entries.size() > 0)
        return true;

    return false;
}
bool COfficeFileFormatChecker::isOfficeFile(const std::wstring & fileName)
{
    //приоритет как оказывается важен
    //Metamorphic Manual for windows 28415.doc
	POLE::Storage storage(fileName.c_str());
    if (storage.open())
    {
        if ( isDocFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_DOC;
            return true;
        }
        else if ( isXlsFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLS;
            return true;
        }
        else if ( isPptFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_PRESENTATION_PPT;
            return true;
        }
        else if ( isMS_OFFCRYPTOFormatFile(&storage) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_OTHER_MS_OFFCRYPTO;
            return true;
        }
    }

    COfficeUtils OfficeUtils(NULL);
    if (OfficeUtils.IsArchive(fileName) == S_OK)
	{
             if ( isOOXFormatFile(fileName) )           return true;
        else if ( isOpenOfficeFormatFile(fileName) )    return true;
        else if ( isOnlyOfficeFormatFile(fileName) )    return true;
        else if ( isXpsFile(fileName) )                 return true;
	}


    // others
    {
		CFile file;
        if (file.OpenFile(std_string2string(fileName)) != S_OK ) return _T("");
		
		unsigned char* buffer = new unsigned char[4096]; //enaf !!
		if (!buffer){file.CloseFile();return _T("");}

		file.ReadFile(buffer,MIN_SIZE_BUFFER);
		int sizeRead = file.GetPosition();
		file.CloseFile();

		if ( isRtfFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_RTF;
		}
		else if ( isBinaryDoctFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_CANVAS_WORD;
		}
		else if ( isBinaryXlstFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_CANVAS_SPREADSHEET;
		}
		else if ( isBinaryPpttFormatFile(buffer,sizeRead) )
		{
			nFileType = AVS_OFFICESTUDIO_FILE_CANVAS_PRESENTATION;
		}
        else if (isHtmlFormatFile(buffer,sizeRead) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_HTML;
        }
        else if (isFB2FormatFile(buffer,sizeRead) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_DOCUMENT_FB2;
        }
        else if (isPdfFormatFile(buffer,sizeRead) )
        {
            nFileType = AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF;
        }
        else if (isDjvuFormatFile(buffer,sizeRead) )
        {
{'fraction_non_alphanumeric': 0.13133239366553354, 'fraction_numerical': 0.01649946107287953, 'mean_word_length': 3.214535290006988, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5088599', 'n_tokens_mistral': 4474, 'n_tokens_neox': 4231, 'n_words': 885}