<commit_before>//  Created by <NAME> Oliora.
//  This code was released to Public domain.

#include "batch_parser.h"
#include <iostream>
#include <fstream>
#include <vector>

namespace
{
    typedef std::vector<char> Buffer;
    
    Buffer readFile(const char *filename)
    {
        std::ifstream is(filename, std::ios::binary);
        
        if (!is)
            throw std::runtime_error("Can not open file");
        
        //is.unsetf(std::ios::skipws);
        
        is.seekg(0, std::ios_base::end);
        
        Buffer res(is.tellg());
        
        is.seekg(0, std::ios_base::beg);
        const size_t realSize = is.rdbuf()->sgetn(&res.front(), res.size()*2);
        res.resize(realSize);
        
        return res;
    }

}


int main(int argc, const char * argv[])
{
    try
    {
        if (argc < 2)
        {
            std::cout << "No filename specified" << std::endl;
            return 2;
        }
    
        const Buffer buf(readFile(argv[1]));
        
        //std::cout.write(&buf.front(), buf.size());
    
        batch_parser::parse(buf.begin(), buf.end());
    }
    catch (const std::exception& ex)
    {
        std::cout << "Error: " << ex.what() << std::endl;
        return 1;
    }
    <commit_msg>Fixed bug when app crashes on empty file. App exit code changed: 1 for parse error and 2 for some internal error<commit_after>//  Created by <NAME> Oliora.
//  This code was released to Public domain.

#include "batch_parser.h"
#include <iostream>
#include <fstream>
#include <vector>

namespace
{
    typedef std::vector<char> Buffer;
    
    Buffer readFile(const char *filename)
    {
        std::ifstream is(filename, std::ios::binary);
        
        if (!is)
            throw std::runtime_error("Can not open file");
        
        //is.unsetf(std::ios::skipws);
        
        is.seekg(0, std::ios_base::end);
        
        Buffer res(is.tellg());

        if (res.size())
        {
            is.seekg(0, std::ios_base::beg);
            const size_t realSize = is.rdbuf()->sgetn(&res.front(), res.size());
            res.resize(realSize);
        }
        
        return res;
    }

}


int main(int argc, const char * argv[])
{
    try
    {
        if (argc < 2)
        {
            std::cout << "No filename specified" << std::endl;
            return 2;
        }
    
        const Buffer buf(readFile(argv[1]));
        
        //std::cout.write(&buf.front(), buf.size());
    
        const bool res = batch_parser::parse(buf.begin(), buf.end());
        return res ? 0 : 1;
    }
    catch (const std::exception& ex)
    {
        std::cout << "Error: " << ex.what() << std::endl;
        return 2;
    }
    
{'fraction_non_alphanumeric': 0.15185185185185185, 'fraction_numerical': 0.006296296296296296, 'mean_word_length': 1.861228813559322, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1010614', 'n_tokens_mistral': 874, 'n_tokens_neox': 859, 'n_words': 204}