<commit_before>#include <iostream>

#include "../../../Common/OfficeFileFormats.h"
#include "../../../Common/OfficeFileFormatChecker.h"

#include "../../../DesktopEditor/graphics/Timer.h"
#include "../../../DesktopEditor/graphics/TemporaryCS.h"
#include "../../../DesktopEditor/common/File.h"
#include "../../../DesktopEditor/common/Directory.h"
#include "../../../DesktopEditor/common/StringBuilder.h"
#include "../../../DesktopEditor/raster/BgraFrame.h"
#include "../../../DesktopEditor/fontengine/ApplicationFontsWorker.h"

#include "../../../OfficeUtils/src/OfficeUtils.h"

#ifdef LINUX
#include <unistd.h>
#include <sys/wait.h>
#include <stdio.h>
#endif

#include <map>

class CConverter;
class CInternalWorker
{
public:
    std::map<int, bool>         m_formats;
    std::vector<std::wstring>   m_files;

    int m_nCount;
    int m_nCurrent;
    int m_nCurrentComplete;

    std::wstring m_sInputFolder;
    std::wstring m_sOutputFolder;

    bool m_bIsStandard;
    bool m_bIsDiffAllInOne;

    NSCriticalSection::CRITICAL_SECTION m_oCS;
    NSCriticalSection::CRITICAL_SECTION m_oCS_OfficeUtils;

public:
    CInternalWorker()
    {
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_DOC, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_ODT, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_RTF, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_TXT, true));

        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_PPSX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_PPT, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_ODP, true));

        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLSX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLS, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_ODS, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_CSV, true));

        m_nCount = 0;
        m_nCurrent = 0;
        m_nCurrentComplete = 0;

        m_bIsDiffAllInOne = true;

        m_oCS.InitializeCriticalSection();
        m_oCS_OfficeUtils.InitializeCriticalSection();
    }
    ~CInternalWorker()
    {
        m_oCS.DeleteCriticalSection();
        m_oCS_OfficeUtils.DeleteCriticalSection();
    }

    CConverter* GetNextConverter();
    void OnConvertFile(CConverter* pConverter, int nCode);
    void Start(int nCores);
    void Cancel();

    void OpenDir(std::wstring sDir)
    {
        m_sInputFolder = sDir;
        std::vector<std::wstring> arFiles = NSDirectory::GetFiles(sDir, true);
        for (std::vector<std::wstring>::iterator iter = arFiles.begin(); iter != arFiles.end(); iter++)
        {
            std::wstring sExt = NSFile::GetFileExtention(*iter);

            if (sExt == L"docx" || sExt == L"doc" || sExt == L"odt" || sExt == L"rtf" ||
                sExt == L"pptx" || sExt == L"ppt" || sExt == L"odp" ||
                sExt == L"xlsx" || sExt == L"xls" || sExt == L"ods")
            {
                m_files.push_back(*iter);
            }
        }
        m_nCount = (int)m_files.size();
    }

    bool IsWork()
    {
        CTemporaryCS oCS(&m_oCS);
        return (m_nCurrentComplete < m_nCount);
    }
};

namespace NSX2T
{
    int Convert(const std::wstring& sConverterPath, const std::wstring sXmlPath)<commit_msg>Add docxf & oform to standardtester<commit_after>#include <iostream>

#include "../../../Common/OfficeFileFormats.h"
#include "../../../Common/OfficeFileFormatChecker.h"

#include "../../../DesktopEditor/graphics/Timer.h"
#include "../../../DesktopEditor/graphics/TemporaryCS.h"
#include "../../../DesktopEditor/common/File.h"
#include "../../../DesktopEditor/common/Directory.h"
#include "../../../DesktopEditor/common/StringBuilder.h"
#include "../../../DesktopEditor/raster/BgraFrame.h"
#include "../../../DesktopEditor/fontengine/ApplicationFontsWorker.h"

#include "../../../OfficeUtils/src/OfficeUtils.h"

#ifdef LINUX
#include <unistd.h>
#include <sys/wait.h>
#include <stdio.h>
#endif

#include <map>

class CConverter;
class CInternalWorker
{
public:
    std::map<int, bool>         m_formats;
    std::vector<std::wstring>   m_files;

    int m_nCount;
    int m_nCurrent;
    int m_nCurrentComplete;

    std::wstring m_sInputFolder;
    std::wstring m_sOutputFolder;

    bool m_bIsStandard;
    bool m_bIsDiffAllInOne;

    NSCriticalSection::CRITICAL_SECTION m_oCS;
    NSCriticalSection::CRITICAL_SECTION m_oCS_OfficeUtils;

public:
    CInternalWorker()
    {
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_DOC, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_ODT, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_RTF, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_TXT, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM, true));

        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_PPSX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_PPT, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_PRESENTATION_ODP, true));

        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLSX, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLS, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_ODS, true));
        m_formats.insert(std::make_pair<int, bool>(AVS_OFFICESTUDIO_FILE_SPREADSHEET_CSV, true));

        m_nCount = 0;
        m_nCurrent = 0;
        m_nCurrentComplete = 0;

        m_bIsDiffAllInOne = true;

        m_oCS.InitializeCriticalSection();
        m_oCS_OfficeUtils.InitializeCriticalSection();
    }
    ~CInternalWorker()
    {
        m_oCS.DeleteCriticalSection();
        m_oCS_OfficeUtils.DeleteCriticalSection();
    }

    CConverter* GetNextConverter();
    void OnConvertFile(CConverter* pConverter, int nCode);
    void Start(int nCores);
    void Cancel();

    void OpenDir(std::wstring sDir)
    {
        m_sInputFolder = sDir;
        std::vector<std::wstring> arFiles = NSDirectory::GetFiles(sDir, true);
        for (std::vector<std::wstring>::iterator iter = arFiles.begin(); iter != arFiles.end(); iter++)
        {
            std::wstring sExt = NSFile::GetFileExtention(*iter);

            if (sExt == L"docx" || sExt == L"doc" || sExt == L"odt" || sExt == L"rtf" || sExt == L"docxf" || sExt == L"oform" ||
                sExt == L"pptx" || sExt == L"ppt" || sExt == L"odp" ||
                sExt == L"xlsx" || sExt == L"xls" || sExt == L"ods")
            {
                m_files.push_back(*iter);
            }
        }
        m_nCount = (int)m_files.size();
    }

    bool IsWork()
    {
        CTemporaryCS oCS(&m_oCS);
        return (m_nCurrentComplete < m_nCount);
    }
};

namespace NSX2T
{
    int Convert(const std::wstring& sConverterPath, const std::wstring sXmlPath)
{'fraction_non_alphanumeric': 0.14729764019284444, 'fraction_numerical': 0.0010149708195889369, 'mean_word_length': 3.56720741599073, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '119444', 'n_tokens_mistral': 2944, 'n_tokens_neox': 2844, 'n_words': 385}