<commit_before>#include <algorithm>
#include <fstream>
#include <iostream>
#include <sstream>
#include <vector>
#include <map>

#include "MSFG.hh"

using namespace std;


MultiStringFactorGraph::~MultiStringFactorGraph() {
    for (auto it = arcs.begin(); it != arcs.end(); ++it)
        delete *it;
}


bool ascending_by_first(pair<int, int> i,pair<int, int> j) { return (i.first < j.first); }

void
MultiStringFactorGraph::add(const FactorGraph &text)
{
    map<fg_node_idx_t, msfg_node_idx_t> visited_nodes;
    multimap<fg_node_idx_t, pair<msfg_node_idx_t, FactorGraph::Arc*> > arcs_to_process;
    // multimap key is target index in fg, value pair source index in msfg, arc in fg
    // ordering arcs by target index ensures topological order for text nodes in msfg

    const FactorGraph::Node &fg_node = text.nodes[0];
    for (auto fg_arcit = fg_node.outgoing.cbegin(); fg_arcit != fg_node.outgoing.cend(); ++fg_arcit)
        arcs_to_process.insert(make_pair((**fg_arcit).target_node, make_pair(0, *fg_arcit)));
    visited_nodes[0] = 0;

    while (arcs_to_process.size() > 0) {

        auto arc_to_process = arcs_to_process.begin();
        FactorGraph::Arc *arc = (*arc_to_process).second.second;
        fg_node_idx_t fg_source_node = arc->source_node;
        fg_node_idx_t fg_target_node = arc->target_node;
        arcs_to_process.erase(arcs_to_process.begin());
        msfg_node_idx_t msfg_source_node = (*arc_to_process).second.first;

        string target_node_factor = text.get_factor(fg_target_node);
        msfg_node_idx_t msfg_target_node = 0;

        // Check if have connected to this node already from some other node
        // Just add arc and continue
        if (visited_nodes.find(fg_target_node) != visited_nodes.end()) {
            msfg_target_node = visited_nodes[fg_target_node];
            find_or_create_arc(msfg_source_node, msfg_target_node);
            continue;
        }

        // Check if node exists, just not connected to it yet
        MultiStringFactorGraph::Node &msfg_node = nodes[msfg_source_node];
        for (auto msfg_arcit = msfg_node.outgoing.begin(); msfg_arcit != msfg_node.outgoing.end(); ++msfg_arcit) {
            if (nodes[(**msfg_arcit).target_node].factor == target_node_factor) {
                msfg_target_node = (**msfg_arcit).target_node;
                break;
            }
        }
        if (msfg_target_node != 0) {
            visited_nodes[fg_target_node] = msfg_target_node;
            const FactorGraph::Node &fg_node = text.nodes[fg_target_node];
            for (auto fg_arcit = fg_node.outgoing.cbegin(); fg_arcit != fg_node.outgoing.cend(); ++fg_arcit)
                arcs_to_process.insert(make_pair((**fg_arcit).target_node, make_pair(msfg_target_node, *fg_arcit)));
            find_or_create_arc(msfg_source_node, msfg_target_node);
            continue;
        }

        // Create new node
        nodes.push_back(Node(target_node_factor));
        msfg_target_node = nodes.size()-1;
        visited_nodes[fg_target_node] = msfg_target_node;
        const FactorGraph::Node &fg_node = text.nodes[fg_target_node];
        for (auto fg_arcit = fg_node.outgoing.cbegin(); fg_arcit != fg_node.outgoing.cend(); ++fg_arcit)
            arcs_to_process.insert(make_pair((**fg_arcit).target_node, make_pair(msfg_target_node, *fg_arcit)));
        create_arc(msfg_source_node, msfg_target_node);
        continue;

    }

    string_end_nodes[text.text] = visited_nodes[text.nodes.size()-1];
}


int
MultiStringFactorGraph::num_paths(const std::string &text) const
{
    if (nodes.size() == 0) return 0;<commit_msg>Cleaned up some code.<commit_after>#include <algorithm>
#include <fstream>
#include <iostream>
#include <sstream>
#include <vector>
#include <map>

#include "MSFG.hh"

using namespace std;


MultiStringFactorGraph::~MultiStringFactorGraph() {
    for (auto it = arcs.begin(); it != arcs.end(); ++it)
        delete *it;
}


bool ascending_by_first(pair<int, int> i,pair<int, int> j) { return (i.first < j.first); }

void
MultiStringFactorGraph::add(const FactorGraph &text)
{
    map<fg_node_idx_t, msfg_node_idx_t> visited_nodes;
    multimap<fg_node_idx_t, pair<msfg_node_idx_t, FactorGraph::Arc*> > arcs_to_process;
    // multimap key is target index in fg, value pair source index in msfg, arc in fg
    // ordering arcs by target index ensures topological order for text nodes in msfg

    const FactorGraph::Node &fg_node = text.nodes[0];
    for (auto fg_arcit = fg_node.outgoing.cbegin(); fg_arcit != fg_node.outgoing.cend(); ++fg_arcit)
        arcs_to_process.insert(make_pair((**fg_arcit).target_node, make_pair(0, *fg_arcit)));

    while (arcs_to_process.size() > 0) {

        auto arc_to_process = arcs_to_process.begin();
        FactorGraph::Arc *arc = (*arc_to_process).second.second;
        fg_node_idx_t fg_source_node = arc->source_node;
        fg_node_idx_t fg_target_node = arc->target_node;
        msfg_node_idx_t msfg_source_node = (*arc_to_process).second.first;
        arcs_to_process.erase(arc_to_process);

        string target_node_factor = text.get_factor(fg_target_node);
        msfg_node_idx_t msfg_target_node = 0;

        // Check if have connected to this node already from some other node
        // Just find or add arc and continue
        if (visited_nodes.find(fg_target_node) != visited_nodes.end()) {
            msfg_target_node = visited_nodes[fg_target_node];
            find_or_create_arc(msfg_source_node, msfg_target_node);
            continue;
        }

        // Check if node exists, just not visited yet
        MultiStringFactorGraph::Node &msfg_node = nodes[msfg_source_node];
        for (auto msfg_arcit = msfg_node.outgoing.begin(); msfg_arcit != msfg_node.outgoing.end(); ++msfg_arcit) {
            if (nodes[(**msfg_arcit).target_node].factor == target_node_factor) {
                msfg_target_node = (**msfg_arcit).target_node;
                break;
            }
        }
        if (msfg_target_node != 0) {
            visited_nodes[fg_target_node] = msfg_target_node;
            const FactorGraph::Node &fg_node = text.nodes[fg_target_node];
            for (auto fg_arcit = fg_node.outgoing.cbegin(); fg_arcit != fg_node.outgoing.cend(); ++fg_arcit)
                arcs_to_process.insert(make_pair((**fg_arcit).target_node, make_pair(msfg_target_node, *fg_arcit)));
            continue;
        }

        // Create new node and arc
        nodes.push_back(Node(target_node_factor));
        msfg_target_node = nodes.size()-1;
        visited_nodes[fg_target_node] = msfg_target_node;
        const FactorGraph::Node &fg_node = text.nodes[fg_target_node];
        for (auto fg_arcit = fg_node.outgoing.cbegin(); fg_arcit != fg_node.outgoing.cend(); ++fg_arcit)
            arcs_to_process.insert(make_pair((**fg_arcit).target_node, make_pair(msfg_target_node, *fg_arcit)));
        create_arc(msfg_source_node, msfg_target_node);
    }

    string_end_nodes[text.text] = visited_nodes[text.nodes.size()-1];
}


int
MultiStringFactorGraph::num_paths(const std::string &text) const
{
    if (nodes.size() == 0) return 0;
{'fraction_non_alphanumeric': 0.11512510542592072, 'fraction_numerical': 0.0028113578858588698, 'mean_word_length': 3.4055727554179565, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1279053', 'n_tokens_mistral': 2497, 'n_tokens_neox': 2457, 'n_words': 453}