<commit_before>#include <boost/lexical_cast.hpp>
#include <boost/asio/io_service.hpp>
#include <boost/bind.hpp>
#include <boost/thread/thread.hpp>
#include <boost/thread.hpp>
#include <boost/log/core.hpp>
#include <boost/log/trivial.hpp>
#include <boost/log/expressions.hpp>
#include <boost/program_options.hpp>
#include <queue>
#include <iostream>
#include <fstream>
#include "frame.h"
#include "target_identifier.h"
#include "imgimport.h"

using namespace std;
using namespace boost;
using namespace cv;
namespace logging = boost::log;
namespace po = boost::program_options;

Frame * next_image();
int handle_args(int argc, char** argv);
queue<Frame *> in_buffer;
queue<Target *> out_buffer;


boost::asio::io_service ioService;
boost::thread_group threadpool;

vector<string> file_names;
int workers = 0;
bool hasMoreFrames = true;

// Processing module classes
ImageImport importer("telemetry.csv");
TargetIdentifier identifier;

void worker(Frame *f)
{
    workers++;
	assert(!f->get_img().empty());
    identifier.process_frame(f);

	workers--;
}

void read_images()
{
    Frame *currentFrame;
    while(hasMoreFrames)
    {
		Frame * f = importer.next_frame();
        if(f){
            in_buffer.push(f);
        } else {
            hasMoreFrames = false;       
            ioService.stop();
        }
        boost::this_thread::sleep(boost::posix_time::milliseconds(30));
    }
}

void assign_workers()
{
	Frame * current;
    while(hasMoreFrames || in_buffer.size() > 0)
    {
        if(in_buffer.size() > 0)
        {
            current = in_buffer.front();
            // spawn worker to process image;
            BOOST_LOG_TRIVIAL(info) << "Working...";
            ioService.post(boost::bind(worker, current));
            in_buffer.pop();
        }

        boost::this_thread::sleep(boost::posix_time::milliseconds(30));
    }
}

void output()
{
    filebuf fb;
    while(hasMoreFrames || out_buffer.size() > 0)
    {
        if(out_buffer.size() > 0)
        {
            fb.open("out.txt", ios::app);
            ostream out(&fb);

            out << out_buffer.front();
            // Output
            out_buffer.pop();
            fb.close();
        }
        boost::this_thread::sleep(boost::posix_time::milliseconds(30));
    }
}

void init()
{
    logging::core::get()->set_filter
    (
        logging::trivial::severity >= logging::trivial::info
    );
}


int main( int argc, char** argv )
{
    init();
    if(handle_args(argc, argv) == 1) return 0;

    int processors = boost::thread::hardware_concurrency();

    ioService.post(boost::bind(read_images));
    ioService.post(boost::bind(assign_workers));
    ioService.post(boost::bind(output));

	boost::asio::io_service::work work(ioService);
    for(int i = 0; i < processors; i++)
    {
        threadpool.create_thread(
            boost::bind(&boost::asio::io_service::run, &ioService)
        );
    }
    threadpool.join_all();
    return 0;
}

int handle_args(int argc, char ** argv){
    po::options_description description("Usage: warg-cv [OPTION]");

    description.add_options()
        ("help,h", "Display this help message")
        ("version,v", "Display the version number")
        ("images,i", po::value<string>(), "Directory containing image files to be processed")
        ("video,v", po::value<int>(), "Video device to capture images from");
    
    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, description), vm);
    po::notify(vm);  

    if (vm.count("help")) {
       cout << description << "\n";
        return 1;
    }

    return 0; 
}<commit_msg>added catch for illegal arguments<commit_after>#include <boost/lexical_cast.hpp>
#include <boost/asio/io_service.hpp>
#include <boost/bind.hpp>
#include <boost/thread/thread.hpp>
#include <boost/thread.hpp>
#include <boost/log/core.hpp>
#include <boost/log/trivial.hpp>
#include <boost/log/expressions.hpp>
#include <boost/program_options.hpp>
#include <queue>
#include <iostream>
#include <fstream>
#include "frame.h"
#include "target_identifier.h"
#include "imgimport.h"

using namespace std;
using namespace boost;
using namespace cv;
namespace logging = boost::log;
namespace po = boost::program_options;

Frame * next_image();
int handle_args(int argc, char** argv);
queue<Frame *> in_buffer;
queue<Target *> out_buffer;


boost::asio::io_service ioService;
boost::thread_group threadpool;

vector<string> file_names;
int workers = 0;
bool hasMoreFrames = true;

// Processing module classes
ImageImport importer("telemetry.csv");
TargetIdentifier identifier;

void worker(Frame *f)
{
    workers++;
	assert(!f->get_img().empty());
    identifier.process_frame(f);

	workers--;
}

void read_images()
{
    Frame *currentFrame;
    while(hasMoreFrames)
    {
		Frame * f = importer.next_frame();
        if(f){
            in_buffer.push(f);
        } else {
            hasMoreFrames = false;       
            ioService.stop();
        }
        boost::this_thread::sleep(boost::posix_time::milliseconds(30));
    }
}

void assign_workers()
{
	Frame * current;
    while(hasMoreFrames || in_buffer.size() > 0)
    {
        if(in_buffer.size() > 0)
        {
            current = in_buffer.front();
            // spawn worker to process image;
            BOOST_LOG_TRIVIAL(info) << "Working...";
            ioService.post(boost::bind(worker, current));
            in_buffer.pop();
        }

        boost::this_thread::sleep(boost::posix_time::milliseconds(30));
    }
}

void output()
{
    filebuf fb;
    while(hasMoreFrames || out_buffer.size() > 0)
    {
        if(out_buffer.size() > 0)
        {
            fb.open("out.txt", ios::app);
            ostream out(&fb);

            out << out_buffer.front();
            // Output
            out_buffer.pop();
            fb.close();
        }
        boost::this_thread::sleep(boost::posix_time::milliseconds(30));
    }
}

void init()
{
    logging::core::get()->set_filter
    (
        logging::trivial::severity >= logging::trivial::info
    );
}


int main( int argc, char** argv )
{
    init();
    if(handle_args(argc, argv) == 1) return 0;

    int processors = boost::thread::hardware_concurrency();

    ioService.post(boost::bind(read_images));
    ioService.post(boost::bind(assign_workers));
    ioService.post(boost::bind(output));

	boost::asio::io_service::work work(ioService);
    for(int i = 0; i < processors; i++)
    {
        threadpool.create_thread(
            boost::bind(&boost::asio::io_service::run, &ioService)
        );
    }
    threadpool.join_all();
    return 0;
}

int handle_args(int argc, char ** argv){
    try {
        po::options_description description("Usage: warg-cv [OPTION]");

        description.add_options()
            ("help,h", "Display this help message")
            ("images,i", po::value<string>(), "Directory containing image files to be processed")
            ("device,d", po::value<int>(), "Video device to capture images from");
    
        po::variables_map vm;
        po::store(po::parse_command_line(argc, argv, description), vm);
        po::notify(vm);  

        if (vm.count("help")) {
            cout << description << "\n";
            return 1;
        }
    } catch(std::exception &e) {
        cout << e.what() << endl;
        return 1;
    }

    return 0; 
}
{'fraction_non_alphanumeric': 0.15327767893800465, 'fraction_numerical': 0.004789927466812645, 'mean_word_length': 2.924812030075188, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1900257', 'n_tokens_mistral': 2388, 'n_tokens_neox': 2295, 'n_words': 463}