<commit_before>#include <cmath>
#include <ctime>
#include "compare_files.hpp"

/*
   TODO:
   -Settings:
     * set default_seq_len
     * only show papers with > n plagiarized phrases
     * set directory to save results
     * ability to save all these settings

   -Add a help button (maybe a question mark) that opens basic info

   -Make standalone program for Linux, Windows, and Mac OS X
*/

int calculate_progress(const int files_seen, const int total_files)
{
    return ceil(100*(static_cast<double>(files_seen)/total_files));
}

template<typename A, typename B>
std::pair<B,A> flip_pair(const std::pair<A,B> &p)
{
    return std::pair<B,A>(p.second, p.first);
}

template<typename A, typename B, template<class,class,class...> class M, class... Args>
std::multimap<B,A> flip_map(const M<A,B,Args...> &src)
{
    std::multimap<B,A> dst;
    std::transform(src.begin(), src.end(),
                   std::inserter(dst, dst.begin()),
                   flip_pair<A,B>);
    return dst;
}

void write_results(const std::string path, std::map<string_pair, unsigned int> common_seqs)
{
    std::ofstream file(get_filename(path));

    file << "Automatically generated by Veritas" << "\n";
    file << "----------------------------------" << "\n";

    std::time_t t = std::time(0);
    file << "Date: " << std::asctime(std::localtime(&t));

    file << "Directory: " << path << "\n\n";

    auto sorted_map = flip_map(common_seqs);

    const int num_seqs_threshold = 10;
    unsigned int num_writes = 0;

    for (auto it = sorted_map.rbegin(); it !=  sorted_map.rend(); ++it) {
        if (it->first > num_seqs_threshold) {
            file << "Files: " << (it->second).first << ", " << (it->second).second << "\n";
            file << "Number of common 6-word sequences: " << it->first << "\n";
            file << "\n";
            ++num_writes;
        }
    }

    if (num_writes == 0) {
        file << "No two files had more than " << num_seqs_threshold
             << " 6-word sequences in common." << "\n";
    }
}


void start_comparisons(Veritas *dialog, string_list file_names, const std::string path)
{
    const int DEFAULT_SEQ_LENGTH = 6;
    const int NUM_FILES = file_names.size();

    if (NUM_FILES <= 1) {
        return;
    }

    std::map<std::string, string_list> seq_map;

    for (auto file_name : file_names) {
        const std::string file_str = file_to_str(file_name);
        string_list seqs = get_sequences(split_str(file_str), DEFAULT_SEQ_LENGTH);
        const std::string fname = clean_file_name(file_name);

        for (auto seq : seqs) {
            if (std::find(seq_map[seq].begin(), seq_map[seq].end(), fname)
                    == seq_map[seq].end()) {
                seq_map[seq].push_back(fname);
            }
        }
    }

    std::map<string_pair, unsigned int> common_seqs;

    for (int i = 0; i != NUM_FILES; ++i) {
        dialog->update_progress_bar(calculate_progress(i, NUM_FILES));
        qApp->processEvents();

        const std::string file_str = file_to_str(file_names[i]);
        string_list seqs = get_sequences(split_str(file_str), DEFAULT_SEQ_LENGTH);
        const std::string fname = clean_file_name(file_names[i]);

        for (auto seq : seqs) {
            for (auto fname_val : seq_map[seq]) {
                if (fname_val != fname &&
                    common_seqs.count(std::make_pair(fname_val, fname)) == 0)
                {
                    ++common_seqs[std::make_pair(fname, fname_val)];
                }
            }
        }
    }

    dialog->update_progress_bar(100);

    write_results(path, common_seqs);
}
<commit_msg>Update compare_files.cpp<commit_after>#include <cmath>
#include <ctime>
#include "compare_files.hpp"

/*
   TODO:
   -Settings:
     * set default_seq_len
     * only show papers with > n plagiarized phrases
     * set directory to save results
     * ability to save all these settings

   -Add a help button (maybe a question mark) that opens basic info

   -Make standalone program for Linux, Windows, and Mac OS X
*/

int calculate_progress(const int files_seen, const int total_files)
{
    return ceil(100*(static_cast<double>(files_seen)/total_files));
}

// source: https://stackoverflow.com/questions/5056645
template<typename A, typename B>
std::pair<B,A> flip_pair(const std::pair<A,B> &p)
{
    return std::pair<B,A>(p.second, p.first);
}

template<typename A, typename B, template<class,class,class...> class M, class... Args>
std::multimap<B,A> flip_map(const M<A,B,Args...> &src)
{
    std::multimap<B,A> dst;
    std::transform(src.begin(), src.end(),
                   std::inserter(dst, dst.begin()),
                   flip_pair<A,B>);
    return dst;
}

void write_results(const std::string path, std::map<string_pair, unsigned int> common_seqs)
{
    std::ofstream file(get_filename(path));

    file << "Automatically generated by Veritas" << "\n";
    file << "----------------------------------" << "\n";

    std::time_t t = std::time(0);
    file << "Date: " << std::asctime(std::localtime(&t));

    file << "Directory: " << path << "\n\n";

    auto sorted_map = flip_map(common_seqs);

    const int num_seqs_threshold = 10;
    unsigned int num_writes = 0;

    for (auto it = sorted_map.rbegin(); it !=  sorted_map.rend(); ++it) {
        if (it->first > num_seqs_threshold) {
            file << "Files: " << (it->second).first << ", " << (it->second).second << "\n";
            file << "Number of common 6-word sequences: " << it->first << "\n";
            file << "\n";
            ++num_writes;
        }
    }

    if (num_writes == 0) {
        file << "No two files had more than " << num_seqs_threshold
             << " 6-word sequences in common." << "\n";
    }
}


void start_comparisons(Veritas *dialog, string_list file_names, const std::string path)
{
    const int DEFAULT_SEQ_LENGTH = 6;
    const int NUM_FILES = file_names.size();

    if (NUM_FILES <= 1) {
        return;
    }

    std::map<std::string, string_list> seq_map;

    for (auto file_name : file_names) {
        const std::string file_str = file_to_str(file_name);
        string_list seqs = get_sequences(split_str(file_str), DEFAULT_SEQ_LENGTH);
        const std::string fname = clean_file_name(file_name);

        for (auto seq : seqs) {
            if (std::find(seq_map[seq].begin(), seq_map[seq].end(), fname)
                    == seq_map[seq].end()) {
                seq_map[seq].push_back(fname);
            }
        }
    }

    std::map<string_pair, unsigned int> common_seqs;

    for (int i = 0; i != NUM_FILES; ++i) {
        dialog->update_progress_bar(calculate_progress(i, NUM_FILES));
        qApp->processEvents();

        const std::string file_str = file_to_str(file_names[i]);
        string_list seqs = get_sequences(split_str(file_str), DEFAULT_SEQ_LENGTH);
        const std::string fname = clean_file_name(file_names[i]);

        for (auto seq : seqs) {
            for (auto fname_val : seq_map[seq]) {
                if (fname_val != fname &&
                    common_seqs.count(std::make_pair(fname_val, fname)) == 0)
                {
                    ++common_seqs[std::make_pair(fname, fname_val)];
                }
            }
        }
    }

    dialog->update_progress_bar(100);

    write_results(path, common_seqs);
}

{'fraction_non_alphanumeric': 0.15386717682426893, 'fraction_numerical': 0.005602623667668762, 'mean_word_length': 2.547746000969462, 'pattern_counts': {'":': 0, '<': 125, '<?xml version=': 0, '>': 53, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4972667', 'n_tokens_mistral': 2495, 'n_tokens_neox': 2375, 'n_words': 537}