<commit_before>/* This file is part Strigi Desktop Search
 *
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>

#include <iostream>
#include <fstream>
#include <string>
#include <set>

#include "XesamQLParser.h"
#include "StrigiQueryBuilder.h"

using namespace std;
using namespace Dijon;
using namespace Strigi;

int main(int argc, char *argv[])
{
    int opt;
    set<string> ulQueries, qlQueries;
    string outfile;

    outfile.clear();
    
    if (argc < 3)
    {
        fprintf(stderr,
                "Usage: %s [-u ulQueryFile] [-q qlQueryFile] [-o outputfile]\n",
                argv[0]);
        exit(EXIT_FAILURE);
    }
    
    while ((opt = getopt(argc, argv, "u:q:o:")) != -1)
    {
        switch (opt) {
            case 'u':
                // user language query
                ulQueries.insert (optarg);
                break;
            case 'q':
                // query language query
                qlQueries.insert (optarg);
                break;
            case 'o':
                outfile = optarg;
                break;
            default: /* ’?’ */
                fprintf(stderr,
                        "Usage: %s [-u ulQueryFile] [-q qlQueryFile]\n",
                        argv[0]);
                exit(EXIT_FAILURE);
        }
    }

    for (set<string>::iterator iter = qlQueries.begin();
         iter != qlQueries.end(); iter++)
    {
        printf ("processing query from file %s\n", (*iter).c_str());
        
        ifstream queryfile;

        queryfile.open ((*iter).c_str(), ifstream::in);

        if (!queryfile.is_open())
        {
            fprintf (stderr, "Unable to open file %s\n", (*iter).c_str());
            continue;
        }
        
        string querytext;

        char buffer [500];
        
        while (!queryfile.eof())
        {
            queryfile.getline (buffer, 500);
            querytext += buffer;
        }

        queryfile.close();
        
        XesamQLParser xesamQlParser;
        StrigiQueryBuilder strigiQueryBuilder;

        if (xesamQlParser.parse ( querytext, strigiQueryBuilder))
        {
            printf ("query parsed successfully\n");
            Query query = strigiQueryBuilder.get_query();
            if (outfile.length() == 0)
                continue;

            ofstream out;
            out.open (outfile.c_str());
            out << query;
            out.close();
        }
        else
            printf ("error parsing query\n");
    }

    for (set<string>::iterator iter = ulQueries.begin();
         iter != ulQueries.end(); iter++)
    {
        printf ("NOT YET IMPLEMENTED\n");
/*        printf ("processing user language query from file %s\n",
                (*iter).c_str());
        
        ifstream queryfile;

        queryfile.open ((*iter).c_str(), ifstream::in);

        if (!queryfile.is_open())
        {
            fprintf (stderr, "Unable to open file %s\n", (*iter).c_str());
            continue;
        }
        
        string querytext;

        char buffer [500];
        
        while (!queryfile.eof())
        {
            queryfile.getline (buffer, 500);
            querytext += buffer;
        }

        queryfile.close();
        
        XesamULParser xesamUlParser;
        StrigiQueryBuilder strigiQueryBuilder;

        if (xesamUlParser.parse ( querytext, strigiQueryBuilder))
        {
            printf ("query parsed successfully\n");
            Strigi::Query query = strigiQueryBuilder.get_query();

            if (outfile.length() == 0)
                continue;

            ofstream out;
            out.open (outfile.c_str());
            out << query;
            out.close();
        }
        else
            printf ("error parsing query\n");*/
    }
    
    return 0;
}<commit_msg>Updated xesam test program, now it handles also userlanguage queries<commit_after>/* This file is part Strigi Desktop Search
 *
 * Copyright (C) 2007 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>

#include <iostream>
#include <fstream>
#include <string>
#include <set>

#include "XesamQLParser.h"
#include "StrigiQueryBuilder.h"
#include "xesam_ul_driver.hh"

using namespace std;
using namespace Dijon;
using namespace Strigi;

int main(int argc, char *argv[])
{
    int opt;
    set<string> ulQueries, qlQueries;
    string outfile;

    outfile.clear();
    
    if (argc < 3)
    {
        fprintf(stderr,
                "Usage: %s [-u ulQueryFile] [-q qlQueryFile] [-o outputfile]\n",
                argv[0]);
        exit(EXIT_FAILURE);
    }
    
    while ((opt = getopt(argc, argv, "u:q:o:")) != -1)
    {
        switch (opt) {
            case 'u':
                // user language query
                ulQueries.insert (optarg);
                break;
            case 'q':
                // query language query
                qlQueries.insert (optarg);
                break;
            case 'o':
                outfile = optarg;
                break;
            default: /* ’?’ */
                fprintf(stderr,
                        "Usage: %s [-u ulQueryFile] [-q qlQueryFile]\n",
                        argv[0]);
                exit(EXIT_FAILURE);
        }
    }

    for (set<string>::iterator iter = qlQueries.begin();
         iter != qlQueries.end(); iter++)
    {
        printf ("processing query from file %s\n", (*iter).c_str());
        
        ifstream queryfile;

        queryfile.open ((*iter).c_str(), ifstream::in);

        if (!queryfile.is_open())
        {
            fprintf (stderr, "Unable to open file %s\n", (*iter).c_str());
            continue;
        }
        
        string querytext;

        char buffer [500];
        
        while (!queryfile.eof())
        {
            queryfile.getline (buffer, 500);
            querytext += buffer;
        }

        queryfile.close();
        
        XesamQLParser xesamQlParser;
        StrigiQueryBuilder strigiQueryBuilder;

        if (xesamQlParser.parse ( querytext, strigiQueryBuilder))
        {
            printf ("query parsed successfully\n");
            Query query = strigiQueryBuilder.get_query();
            if (outfile.length() == 0)
                continue;

            ofstream out;
            out.open (outfile.c_str());
            out << query;
            out.close();
        }
        else
            printf ("error parsing query\n");
    }

    
    for (set<string>::iterator iter = ulQueries.begin();
         iter != ulQueries.end(); iter++)
    {
        printf ("processing user language query from file %s\n",
                (*iter).c_str());

        XesamUlDriver driver;
        driver.parseFile (*iter);
        Query* query = driver.query();

        if (!outfile.empty()) {
            ofstream out;

            out.open (outfile.c_str());
            if (out.is_open()) {
                out << *query;
                out.close();
            }
            else
              cerr << "unable to write to file " << outfile << endl;
        }
    }
    
    return 0;
}
{'fraction_non_alphanumeric': 0.11327085502718963, 'fraction_numerical': 0.007289135716765012, 'mean_word_length': 1.7493638676844783, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2116765', 'n_tokens_mistral': 2509, 'n_tokens_neox': 2402, 'n_words': 738}