<commit_before>/*  Copyright (c) MediaArea.net SARL. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license that can
 *  be found in the License.html file in the root of the source tree.
 */

//---------------------------------------------------------------------------
// Pre-compilation
#include "MediaInfo/PreComp.h"
#ifdef __BORLANDC__
    #pragma hdrstop
#endif
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#include "MediaInfo/MediaInfoList_Internal.h"
#include "MediaInfo/MediaInfo_Config.h"
#if defined(MEDIAINFO_FILE_YES)
#include "ZenLib/File.h"
#endif //defined(MEDIAINFO_FILE_YES)
#if defined(MEDIAINFO_DIRECTORY_YES)
#include "ZenLib/Dir.h"
#endif //defined(MEDIAINFO_DIRECTORY_YES)
#include "MediaInfo/Reader/Reader_Directory.h"
#include "MediaInfo/File__Analyse_Automatic.h"
#include <algorithm>
using namespace ZenLib;
using namespace std;
//---------------------------------------------------------------------------

namespace MediaInfoLib
{

//---------------------------------------------------------------------------
extern MediaInfo_Config Config;
//---------------------------------------------------------------------------

//***************************************************************************
// Gestion de la classe
//***************************************************************************

//---------------------------------------------------------------------------
//Constructeurs
MediaInfoList_Internal::MediaInfoList_Internal(size_t Count_Init)
: Thread()
{
    CriticalSectionLocker CSL(CS);

    //Initialisation
    Info.reserve(Count_Init);
    for (size_t Pos=0; Pos<Info.size(); Pos++)
        Info[Pos]=NULL;
    ToParse_AlreadyDone=0;
    ToParse_Total=0;
    CountValid=0;

    //Threading
    BlockMethod=0;
    State=0;
    IsInThread=false;
}

//---------------------------------------------------------------------------
//Destructeur
MediaInfoList_Internal::~MediaInfoList_Internal()
{
    Close();
}

//***************************************************************************
// Fichiers
//***************************************************************************

//---------------------------------------------------------------------------
size_t MediaInfoList_Internal::Open(const String &File_Name, const fileoptions_t Options)
{
    //Option FileOption_Close
    if (Options & FileOption_CloseAll)
        Close(All);

    //Option Recursive
    //TODO

    //Get all filenames
    ZtringList List;
    size_t Pos=File_Name.find(__T(':'));
    if (Pos!=string::npos && Pos!=1)
        List.push_back(File_Name);
    #if defined(MEDIAINFO_FILE_YES)
    else if (File::Exists(File_Name))
        List.push_back(File_Name);
    #endif //defined(MEDIAINFO_FILE_YES)
    #if defined(MEDIAINFO_DIRECTORY_YES)
    else
    {
        List=Dir::GetAllFileNames(File_Name, (Options&FileOption_NoRecursive)?Dir::Include_Files:((Dir::dirlist_t)(Dir::Include_Files|Dir::Parse_SubDirs)));
        sort(List.begin(), List.end());
        if (List.empty())
            List.push_back(File_Name); // Try directly the file name e.g. "-" for pipe

        #if MEDIAINFO_ADVANCED
            if (MediaInfoLib::Config.ParseOnlyKnownExtensions_IsSet())
            {
                set<Ztring> ExtensionsList=MediaInfoLib::Config.ParseOnlyKnownExtensions_GetList_Set();
                bool AcceptNoExtension=ExtensionsList.find(Ztring())!=ExtensionsList.end();
                for (size_t i=List.size()-1; i!=(size_t)-1; i--)
                {
                    const Ztring& Name=List[i];
                    size_t Extension_Pos=Name.rfind(__T('.'));
                    if (Extension_Pos!=string::npos && ExtensionsList.find(Name.substr(Extension_Pos+1))==ExtensionsList.end()
                     || Extension_Pos==string::npos && !AcceptNoExtension)
                            List.erase(List.begin()+i);
                }
            }
        #endif //MEDIAINFO_ADVANCED
    }
    #endif //defined(MEDIAINFO_DIRECTORY_YES)

    #if defined(MEDIAINFO_DIRECTORY_YES)
        Reader_Directory().Directory_Cleanup(List);
    #endif //defined(MEDIAINFO_DIRECTORY_YES)

    //Registering files
    {
    CriticalSectionLocker CSL(CS);
    if (ToParse.empty())
        CountValid=0;<commit_msg>Simplify the logic in MediaInfoList_Internal::Open<commit_after>/*  Copyright (c) MediaArea.net SARL. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license that can
 *  be found in the License.html file in the root of the source tree.
 */

//---------------------------------------------------------------------------
// Pre-compilation
#include "MediaInfo/PreComp.h"
#ifdef __BORLANDC__
    #pragma hdrstop
#endif
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#include "MediaInfo/MediaInfoList_Internal.h"
#include "MediaInfo/MediaInfo_Config.h"
#if defined(MEDIAINFO_FILE_YES)
#include "ZenLib/File.h"
#endif //defined(MEDIAINFO_FILE_YES)
#if defined(MEDIAINFO_DIRECTORY_YES)
#include "ZenLib/Dir.h"
#endif //defined(MEDIAINFO_DIRECTORY_YES)
#include "MediaInfo/Reader/Reader_Directory.h"
#include "MediaInfo/File__Analyse_Automatic.h"
#include <algorithm>
using namespace ZenLib;
using namespace std;
//---------------------------------------------------------------------------

namespace MediaInfoLib
{

//---------------------------------------------------------------------------
extern MediaInfo_Config Config;
//---------------------------------------------------------------------------

//***************************************************************************
// Gestion de la classe
//***************************************************************************

//---------------------------------------------------------------------------
//Constructeurs
MediaInfoList_Internal::MediaInfoList_Internal(size_t Count_Init)
: Thread()
{
    CriticalSectionLocker CSL(CS);

    //Initialisation
    Info.reserve(Count_Init);
    for (size_t Pos=0; Pos<Info.size(); Pos++)
        Info[Pos]=NULL;
    ToParse_AlreadyDone=0;
    ToParse_Total=0;
    CountValid=0;

    //Threading
    BlockMethod=0;
    State=0;
    IsInThread=false;
}

//---------------------------------------------------------------------------
//Destructeur
MediaInfoList_Internal::~MediaInfoList_Internal()
{
    Close();
}

//***************************************************************************
// Fichiers
//***************************************************************************

//---------------------------------------------------------------------------
size_t MediaInfoList_Internal::Open(const String &File_Name, const fileoptions_t Options)
{
    //Option FileOption_Close
    if (Options & FileOption_CloseAll)
        Close(All);

    //Option Recursive
    //TODO

    //Get all filenames
    ZtringList List;
    #if defined(MEDIAINFO_DIRECTORY_YES)
    if (Dir::Exists(File_Name))
    {
        List=Dir::GetAllFileNames(File_Name, (Options&FileOption_NoRecursive)?Dir::Include_Files:((Dir::dirlist_t)(Dir::Include_Files|Dir::Parse_SubDirs)));
        sort(List.begin(), List.end());

        #if MEDIAINFO_ADVANCED
            if (MediaInfoLib::Config.ParseOnlyKnownExtensions_IsSet())
            {
                set<Ztring> ExtensionsList=MediaInfoLib::Config.ParseOnlyKnownExtensions_GetList_Set();
                bool AcceptNoExtension=ExtensionsList.find(Ztring())!=ExtensionsList.end();
                for (size_t i=List.size()-1; i!=(size_t)-1; i--)
                {
                    const Ztring& Name=List[i];
                    size_t Extension_Pos=Name.rfind(__T('.'));
                    if (Extension_Pos!=string::npos && ExtensionsList.find(Name.substr(Extension_Pos+1))==ExtensionsList.end()
                     || Extension_Pos==string::npos && !AcceptNoExtension)
                            List.erase(List.begin()+i);
                }
            }
        #endif //MEDIAINFO_ADVANCED
    }
    else
    #endif //defined(MEDIAINFO_DIRECTORY_YES)
    {
        List.push_back(File_Name);
    }

    #if defined(MEDIAINFO_DIRECTORY_YES)
        Reader_Directory().Directory_Cleanup(List);
    #endif //defined(MEDIAINFO_DIRECTORY_YES)

    //Registering files
    {
    CriticalSectionLocker CSL(CS);
    if (ToParse.empty())
        CountValid=0;
{'fraction_non_alphanumeric': 0.34826883910386963, 'fraction_numerical': 0.0023761031907671417, 'mean_word_length': 1.6907153729071538, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2223902', 'n_tokens_mistral': 2287, 'n_tokens_neox': 2197, 'n_words': 375}