<commit_before>
/*

    poedit, a wxWindows i18n catalogs editor

    ---------------
      catalog.cpp
    
      Translations catalog
    
      (c) <NAME>, 1999-2003

*/


#ifdef __GNUG__
#pragma implementation
#endif

#include <wx/wxprec.h>

#include <stdio.h>
#include <wx/utils.h>
#include <wx/tokenzr.h>
#include <wx/log.h>
#include <wx/intl.h>
#include <wx/datetime.h>
#include <wx/config.h>
#include <wx/textfile.h>
#include <wx/strconv.h>
#include <wx/msgdlg.h>

#include "catalog.h"
#include "digger.h"
#include "gexecute.h"
#include "progressinfo.h"
#include "summarydlg.h"



#include "wx/arrimpl.cpp"
WX_DEFINE_OBJARRAY(CatalogDataArray)


// textfile processing utilities:

// Read one line from file, remove all \r and \n characters, ignore empty lines
static wxString ReadTextLine(wxTextFile* f)
{
    wxString s;
    
    while (s.IsEmpty())
    {
        if (f->Eof()) return wxEmptyString;
        s = f->GetNextLine();
    }
    return s;
}


// If input begins with pattern, fill output with end of input (without
// pattern; strips trailing spaces) and return true.  Return false otherwise
// and don't touch output
static bool ReadParam(const wxString& input, const wxString& pattern, wxString& output)
{
    if (input.Length() < pattern.Length()) return false;
    
    for (unsigned i = 0; i < pattern.Length(); i++)
        if (input[i] != pattern[i]) return false;
    output = input.Mid(pattern.Length()).Strip(wxString::trailing);
    return true;
}
        
        
// parsers:

void CatalogParser::Parse()
{
    if (m_textFile->GetLineCount() == 0) return;

    wxString line, dummy;
    wxString mflags, mstr, mtrans, mcomment;
    wxArrayString mrefs;
    unsigned mlinenum;
    
    line = m_textFile->GetFirstLine();
    if (line.IsEmpty()) line = ReadTextLine(m_textFile);

    while (!line.IsEmpty())
    {
        // flags:
        if (ReadParam(line, _T("#, "), dummy))
        {
            mflags = _T("#, ") + dummy;
            line = ReadTextLine(m_textFile);
        }
        
        // references:
        if (ReadParam(line, _T("#: "), dummy))
        {
            wxStringTokenizer tkn(dummy, _T("\t\r\n "));
            while (tkn.HasMoreTokens())
                mrefs.Add(tkn.GetNextToken());
            line = ReadTextLine(m_textFile);
        }
        
        // msgid:
        else if (ReadParam(line, _T("msgid \""), dummy))
        {
            mstr = dummy.RemoveLast();
            mlinenum = m_textFile->GetCurrentLine() + 1;
            while (!(line = ReadTextLine(m_textFile)).IsEmpty())
            {
                if (line[0u] == _T('"') && line.Last() == _T('"'))
                    mstr += line.Mid(1, line.Length() - 2);
                else break;
            }
        }
        
        // msgstr:
        else if (ReadParam(line, _T("msgstr \""), dummy))
        {
            mtrans = dummy.RemoveLast();
            while (!(line = ReadTextLine(m_textFile)).IsEmpty())
            {
                if (line[0u] == _T('"') && line.Last() == _T('"'))
                    mtrans += line.Mid(1, line.Length() - 2);
                else break;
            }
<commit_msg>fix hangups on broken files or on files correct but weird<commit_after>
/*

    poedit, a wxWindows i18n catalogs editor

    ---------------
      catalog.cpp
    
      Translations catalog
    
      (c) <NAME>, 1999-2003

*/


#ifdef __GNUG__
#pragma implementation
#endif

#include <wx/wxprec.h>

#include <stdio.h>
#include <wx/utils.h>
#include <wx/tokenzr.h>
#include <wx/log.h>
#include <wx/intl.h>
#include <wx/datetime.h>
#include <wx/config.h>
#include <wx/textfile.h>
#include <wx/strconv.h>
#include <wx/msgdlg.h>

#include "catalog.h"
#include "digger.h"
#include "gexecute.h"
#include "progressinfo.h"
#include "summarydlg.h"



#include "wx/arrimpl.cpp"
WX_DEFINE_OBJARRAY(CatalogDataArray)


// textfile processing utilities:

// Read one line from file, remove all \r and \n characters, ignore empty lines
static wxString ReadTextLine(wxTextFile* f)
{
    wxString s;
    
    while (s.IsEmpty())
    {
        if (f->Eof()) return wxEmptyString;
        s = f->GetNextLine();
    }
    return s;
}


// If input begins with pattern, fill output with end of input (without
// pattern; strips trailing spaces) and return true.  Return false otherwise
// and don't touch output
static bool ReadParam(const wxString& input, const wxString& pattern, wxString& output)
{
    if (input.Length() < pattern.Length()) return false;
    
    for (unsigned i = 0; i < pattern.Length(); i++)
        if (input[i] != pattern[i]) return false;
    output = input.Mid(pattern.Length()).Strip(wxString::trailing);
    return true;
}
        
        
// parsers:

void CatalogParser::Parse()
{
    if (m_textFile->GetLineCount() == 0) return;

    wxString line, dummy;
    wxString mflags, mstr, mtrans, mcomment;
    wxArrayString mrefs;
    unsigned mlinenum;
    
    line = m_textFile->GetFirstLine();
    if (line.IsEmpty()) line = ReadTextLine(m_textFile);

    while (!line.IsEmpty())
    {
        // ignore empty special tags:
        while (line == _T("#,") || line == _T("#:"))
            line = ReadTextLine(m_textFile);
        
        // flags:        
        if (ReadParam(line, _T("#, "), dummy))
        {
            mflags = _T("#, ") + dummy;
            line = ReadTextLine(m_textFile);
        }
        
        // references:
        if (ReadParam(line, _T("#: "), dummy))
        {
            wxStringTokenizer tkn(dummy, _T("\t\r\n "));
            while (tkn.HasMoreTokens())
                mrefs.Add(tkn.GetNextToken());
            line = ReadTextLine(m_textFile);
        }
        
        // msgid:
        else if (ReadParam(line, _T("msgid \""), dummy))
        {
            mstr = dummy.RemoveLast();
            mlinenum = m_textFile->GetCurrentLine() + 1;
            while (!(line = ReadTextLine(m_textFile)).IsEmpty())
            {
                if (line[0u] == _T('"') && line.Last() == _T('"'))
                    mstr += line.Mid(1, line.Length() - 2);
                else
                    break;
            }
        }
        
        // msgstr:
        else if (ReadParam(line, _T("msgstr \""), dummy))
        {
            mtrans = dummy.RemoveLast();
            while (!(line = ReadTextLine(m_textFile)).IsEmpty())
            {
                if (line[0u] == _T('"') && line.Last() == _T('"'))
                    mtrans += line.Mid(1, line.Length() - 2);
                else
                    break;
            }

{'fraction_non_alphanumeric': 0.1515522251108732, 'fraction_numerical': 0.00581128612937758, 'mean_word_length': 2.005514705882353, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2812479', 'n_tokens_mistral': 2183, 'n_tokens_neox': 2088, 'n_words': 501}