<commit_before>// File__ReferenceFilesHelper - class for analyzing/demuxing reference files
// Copyright (C) 2011-2012 MediaArea.net SARL, <EMAIL>
//
// This library is free software: you can redistribute it and/or modify it
// under the terms of the GNU Library General Public License as published by
// the Free Software Foundation, either version 2 of the License, or
// any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public License
// along with this library. If not, see <http://www.gnu.org/licenses/>.
//
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

//---------------------------------------------------------------------------
// Pre-compilation
#include "MediaInfo/PreComp.h"
#ifdef __BORLANDC__
    #pragma hdrstop
#endif
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#include "MediaInfo/Setup.h"
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#if defined(MEDIAINFO_REFERENCES_YES)
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#include "MediaInfo/Multiple/File__ReferenceFilesHelper.h"
#include "MediaInfo/MediaInfo_Internal.h"
#include "ZenLib/Dir.h"
#include "ZenLib/File.h"
#include "ZenLib/FileName.h"
#include "ZenLib/Format/Http/Http_Utils.h"
#include <set>
#include <algorithm>
#if MEDIAINFO_EVENTS
    #include "MediaInfo/MediaInfo_Events_Internal.h"
#endif //MEDIAINFO_EVENTS
//---------------------------------------------------------------------------

namespace MediaInfoLib
{

//***************************************************************************
// Constructor/Destructor
//***************************************************************************

//---------------------------------------------------------------------------
File__ReferenceFilesHelper::File__ReferenceFilesHelper(File__Analyze* MI_, MediaInfo_Config_MediaInfo* Config_)
{
    //Temp
    MI=MI_;
    Config=Config_;
    Reference=References.end();
    Init_Done=false;
    TestContinuousFileNames=false;
    ContainerHasNoId=false;
    FrameRate=0;
    Duration=0;
    #if MEDIAINFO_NEXTPACKET
        DTS_Interval=(int64u)-1;
    #endif //MEDIAINFO_NEXTPACKET
    #if MEDIAINFO_EVENTS
        StreamID_Previous=(int64u)-1;
    #endif //MEDIAINFO_EVENTS
}

//---------------------------------------------------------------------------
File__ReferenceFilesHelper::~File__ReferenceFilesHelper()
{
    for (references::iterator ReferenceTemp=References.begin(); ReferenceTemp!=References.end(); ++ReferenceTemp)
        delete ReferenceTemp->MI; //ReferenceTemp->MI=NULL;
}

//***************************************************************************
// Streams management
//***************************************************************************

//---------------------------------------------------------------------------
bool File__ReferenceFilesHelper_Algo1 (const File__ReferenceFilesHelper::reference &Ref1, const File__ReferenceFilesHelper::reference &Ref2) { return (Ref1.StreamID<Ref2.StreamID);}
bool File__ReferenceFilesHelper_Algo2 (const File__ReferenceFilesHelper::reference &Ref1, const File__ReferenceFilesHelper::reference &Ref2) { return (Ref1.StreamPos<Ref2.StreamPos);}
bool File__ReferenceFilesHelper_Algo3 (const File__ReferenceFilesHelper::reference &Ref1, const File__ReferenceFilesHelper::reference &Ref2) { return (Ref1.StreamKind<Ref2.StreamKind);}
void File__ReferenceFilesHelper::ParseReferences()
{
    if (!Init_Done)
    {
        if (MI->Config->File_Filter_Audio_Get())
            for (size_t Pos=0; Pos<References.size(); Pos++)
                if (References[Pos].StreamKind!=Stream_Audio)
                {
                    References.erase(References.begin()+Pos);
                    Pos--;
                }

        //Testing IDs
        std::sort(References.begin(), References.end(), File__ReferenceFilesHelper_Algo1);
        std::sort(References.begin(), References.end(), File__ReferenceFilesHelper_Algo2);
        std::sort(References.begin(), References.end(), File__ReferenceFilesHelper_Algo3);
        std::set<int64u> StreamList;
        bool StreamList_DuplicatedIds=false;
        for (Reference=References.begin(); Reference<References.end(); ++Reference)
            if (StreamList.find((*Reference).StreamID)==StreamList.end())
                StreamList.insert((*Reference).StreamID);
            else
            {
                StreamList_DuplicatedIds=true;
                break;
            }
        if (StreamList_DuplicatedIds)
            for (Reference=References.begin(); Reference<References.end(); ++Reference)
                (*Reference).StreamID=Reference-References.begin()+1;

        //Configuring file names
        Reference=References.begin();
        while (Reference!=References.end())
        {
            ZtringList Names=Reference->FileNames;
            ZtringList AbsoluteNames; AbsoluteNames.Separator_Set(0, ",");
            for (size_t Pos=0; Pos<Names.size(); Pos++)
            {<commit_msg>x Compilation fails with some defines<commit_after>// File__ReferenceFilesHelper - class for analyzing/demuxing reference files
// Copyright (C) 2011-2012 MediaArea.net SARL, <EMAIL>
//
// This library is free software: you can redistribute it and/or modify it
// under the terms of the GNU Library General Public License as published by
// the Free Software Foundation, either version 2 of the License, or
// any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public License
// along with this library. If not, see <http://www.gnu.org/licenses/>.
//
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

//---------------------------------------------------------------------------
// Pre-compilation
#include "MediaInfo/PreComp.h"
#ifdef __BORLANDC__
    #pragma hdrstop
#endif
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#include "MediaInfo/Setup.h"
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#if defined(MEDIAINFO_REFERENCES_YES)
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
#include "MediaInfo/Multiple/File__ReferenceFilesHelper.h"
#include "MediaInfo/MediaInfo_Internal.h"
#include "ZenLib/Dir.h"
#include "ZenLib/File.h"
#include "ZenLib/FileName.h"
#include "ZenLib/Format/Http/Http_Utils.h"
#include <set>
#include <algorithm>
#if MEDIAINFO_EVENTS
    #include "MediaInfo/MediaInfo_Events_Internal.h"
#endif //MEDIAINFO_EVENTS
//---------------------------------------------------------------------------

namespace MediaInfoLib
{

//***************************************************************************
// Constructor/Destructor
//***************************************************************************

//---------------------------------------------------------------------------
File__ReferenceFilesHelper::File__ReferenceFilesHelper(File__Analyze* MI_, MediaInfo_Config_MediaInfo* Config_)
{
    //Temp
    MI=MI_;
    Config=Config_;
    Reference=References.end();
    Init_Done=false;
    TestContinuousFileNames=false;
    ContainerHasNoId=false;
    FrameRate=0;
    Duration=0;
    #if MEDIAINFO_NEXTPACKET
        DTS_Interval=(int64u)-1;
    #endif //MEDIAINFO_NEXTPACKET
    #if MEDIAINFO_EVENTS
        StreamID_Previous=(int64u)-1;
    #endif //MEDIAINFO_EVENTS
}

//---------------------------------------------------------------------------
File__ReferenceFilesHelper::~File__ReferenceFilesHelper()
{
    for (references::iterator ReferenceTemp=References.begin(); ReferenceTemp!=References.end(); ++ReferenceTemp)
        delete ReferenceTemp->MI; //ReferenceTemp->MI=NULL;
}

//***************************************************************************
// Streams management
//***************************************************************************

//---------------------------------------------------------------------------
bool File__ReferenceFilesHelper_Algo1 (const File__ReferenceFilesHelper::reference &Ref1, const File__ReferenceFilesHelper::reference &Ref2) { return (Ref1.StreamID<Ref2.StreamID);}
bool File__ReferenceFilesHelper_Algo2 (const File__ReferenceFilesHelper::reference &Ref1, const File__ReferenceFilesHelper::reference &Ref2) { return (Ref1.StreamPos<Ref2.StreamPos);}
bool File__ReferenceFilesHelper_Algo3 (const File__ReferenceFilesHelper::reference &Ref1, const File__ReferenceFilesHelper::reference &Ref2) { return (Ref1.StreamKind<Ref2.StreamKind);}
void File__ReferenceFilesHelper::ParseReferences()
{
    if (!Init_Done)
    {
        #if MEDIAINFO_FILTER
            if (MI->Config->File_Filter_Audio_Get())
                for (size_t Pos=0; Pos<References.size(); Pos++)
                    if (References[Pos].StreamKind!=Stream_Audio)
                    {
                        References.erase(References.begin()+Pos);
                        Pos--;
                    }
        #endif //MEDIAINFO_FILTER

        //Testing IDs
        std::sort(References.begin(), References.end(), File__ReferenceFilesHelper_Algo1);
        std::sort(References.begin(), References.end(), File__ReferenceFilesHelper_Algo2);
        std::sort(References.begin(), References.end(), File__ReferenceFilesHelper_Algo3);
        std::set<int64u> StreamList;
        bool StreamList_DuplicatedIds=false;
        for (Reference=References.begin(); Reference<References.end(); ++Reference)
            if (StreamList.find((*Reference).StreamID)==StreamList.end())
                StreamList.insert((*Reference).StreamID);
            else
            {
                StreamList_DuplicatedIds=true;
                break;
            }
        if (StreamList_DuplicatedIds)
            for (Reference=References.begin(); Reference<References.end(); ++Reference)
                (*Reference).StreamID=Reference-References.begin()+1;

        //Configuring file names
        Reference=References.begin();
        while (Reference!=References.end())
        {
            ZtringList Names=Reference->FileNames;
            ZtringList AbsoluteNames; AbsoluteNames.Separator_Set(0, ",");
            for (size_t Pos=0; Pos<Names.size(); Pos++)
            {
{'fraction_non_alphanumeric': 0.3136255820082579, 'fraction_numerical': 0.007203724852850742, 'mean_word_length': 2.103598691384951, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7055982', 'n_tokens_mistral': 2770, 'n_tokens_neox': 2599, 'n_words': 561}