<commit_before>/* Copyright (C) 2005, 2006, 2007, 2008
 * <NAME>, <NAME>, <NAME>,
 * Universitaet Karlsruhe (TH)
 *
 * This file is part of Contraction Hierarchies.
 *
 * Contraction Hierarchies is free software; you can redistribute it
 * and/or modify it under the terms of the GNU Affero General Public License
 * as published by the Free Software Foundation; either version 3 of
 * the License, or (at your option) any later version.
 *
 * Contraction Hierarchies is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Contraction Hierarchies; see the file COPYING; if not,
 * see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <iomanip>
#include <fstream>
#include <sys/types.h>
#include <fcntl.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>

using namespace std;
using namespace google::protobuf::io;

/**
 * Performing the bucket scans is a crucial part of the many-to-many computation.
 * It can be switched off to allow time measurements of the forward search
 * without accounting for the bucket scans.
 */
const bool performBucketScans = true;


#include "../config.h"
#include "../stats/utils.h"
Counter counter;
//#include "../datastr/general.h"
#include "../datastr/graph/graph.h"
#include "../datastr/graph/SearchGraph.h"

typedef datastr::graph::SearchGraph TransitGraph;

//#include "../stats/log.h"
#include "../processing/DijkstraCH.h"
#include "manyToMany.h"
#include "protos/ch/many2many.pb.h"

typedef datastr::graph::SearchGraph MyGraph;
typedef DijkstraCHManyToManyFW DijkstraManyToManyFW;
typedef DijkstraCHManyToManyBW DijkstraManyToManyBW;

static void readNodes(const MyGraph *const g, const google::protobuf::RepeatedField<uint32_t>& in, vector<NodeID>& vs) {
    vs.resize(in.size());
    for (auto i=0; i < in.size(); i++) {
        // map the actual node ID to the node ID that is used internally by
        // highway-node routing
        vs[i] = g->mapExtToIntNodeID(in.Get(i));
    }
}

static void readMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = open(filename.c_str(), O_RDONLY);
    FileInputStream raw_input(fd);
    raw_input.SetCloseOnDelete(true);
    message.ParseFromZeroCopyStream(&raw_input);
}

static void writeMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = open(filename.c_str(), O_WRONLY);
    FileOutputStream raw_output(fd);
    raw_output.SetCloseOnDelete(true);
    message.SerializeToZeroCopyStream(&raw_output);
}

MyGraph *const readGraph(const string& inGraphFn) {
    ifstream inGraph(inGraphFn, ios::binary);
    if (!inGraph) {
        cerr << "Input file '" << inGraphFn << "' couldn't be opened." << endl;
        exit(-1);
    }
    return new MyGraph(inGraph);
}


/** The main program. */
int main(int argc, char *argv[])
{
    // read arguments
    if (argc < 2) {
        cerr << "Too few arguments!" << endl
             << "Usage: ./manyToMany <inSgrGraph> <inNodes> <outMatrix>" << endl;
        exit(-1);
    }

    const string inGraphFn   = argv[1];
    const string inNodesFn   = argv[2];
    const string outMatrixFn = argv[3];

    const bool writeSourceNodes = false;
    const bool writeMatrix = false;
    const bool validateResult = false;

    VERBOSE( cerr << "read graph from '" << inGraphFn << "'" << endl);
    VERBOSE( cerr << "read nodes from '" << inNodesFn << "'" << endl);
    VERBOSE( cerr << "write matrix to '" << outMatrixFn << "'" << endl);
    VERBOSE( if (writeSourceNodes) cerr << "write source nodes to cerr" << endl );
    VERBOSE( if (writeMatrix)      cerr << "write matrix to cerr" << endl );
    VERBOSE( if (validateResult)   cerr << "validate result" << endl );

    uint32_t sz_sources, sz_targets;
    cin >> sz_sources >> sz_targets;

    // read graph
    MyGraph *const graph = readGraph(inGraphFn);

    VERBOSE( cerr << "Graph read." << endl );

    // read the input
    protos::ch::Nodes nodes;
    readMessage(inNodesFn, nodes);

    vector<NodeID> sources;
    readNodes(graph, nodes.sources(), sources);

    vector<NodeID> targets;
    readNodes(graph, nodes.targets(), targets);

    // create many-to-many object
    LevelID earlyStopLevel = 10;
    ManyToMany<MyGraph, DijkstraManyToManyFW, DijkstraManyToManyBW, performBucketScans> mtm(graph, earlyStopLevel);<commit_msg>Tweaks in I/O.<commit_after>/* Copyright (C) 2005, 2006, 2007, 2008
 * <NAME>, <NAME>, <NAME>,
 * Universitaet Karlsruhe (TH)
 *
 * This file is part of Contraction Hierarchies.
 *
 * Contraction Hierarchies is free software; you can redistribute it
 * and/or modify it under the terms of the GNU Affero General Public License
 * as published by the Free Software Foundation; either version 3 of
 * the License, or (at your option) any later version.
 *
 * Contraction Hierarchies is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Contraction Hierarchies; see the file COPYING; if not,
 * see <http://www.gnu.org/licenses/>.
 */

#include <iostream>
#include <iomanip>
#include <fstream>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>

using namespace std;
using namespace google::protobuf::io;

/**
 * Performing the bucket scans is a crucial part of the many-to-many computation.
 * It can be switched off to allow time measurements of the forward search
 * without accounting for the bucket scans.
 */
const bool performBucketScans = true;


#include "../config.h"
#include "../stats/utils.h"
Counter counter;
//#include "../datastr/general.h"
#include "../datastr/graph/graph.h"
#include "../datastr/graph/SearchGraph.h"

typedef datastr::graph::SearchGraph TransitGraph;

//#include "../stats/log.h"
#include "../processing/DijkstraCH.h"
#include "manyToMany.h"
#include "protos/ch/many2many.pb.h"

typedef datastr::graph::SearchGraph MyGraph;
typedef DijkstraCHManyToManyFW DijkstraManyToManyFW;
typedef DijkstraCHManyToManyBW DijkstraManyToManyBW;

static void readNodes(const MyGraph *const g, const google::protobuf::RepeatedField<uint32_t>& in, vector<NodeID>& vs) {
    vs.resize(in.size());
    for (auto i=0; i < in.size(); i++) {
        // map the actual node ID to the node ID that is used internally by
        // highway-node routing
        vs[i] = g->mapExtToIntNodeID(in.Get(i));
    }
}

static void readMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = open(filename.c_str(), O_RDONLY);
    message.ParseFromFileDescriptor(fd);
    close(fd);
}

static void writeMessage(const string& filename, ::google::protobuf::Message& message) {
    int fd = creat(filename.c_str(), 0644);
    message.SerializeToFileDescriptor(fd);
    close(fd);
}

MyGraph *const readGraph(const string& inGraphFn) {
    ifstream inGraph(inGraphFn, ios::binary);
    if (!inGraph) {
        cerr << "Input file '" << inGraphFn << "' couldn't be opened." << endl;
        exit(-1);
    }
    return new MyGraph(inGraph);
}


/** The main program. */
int main(int argc, char *argv[])
{
    // read arguments
    if (argc < 2) {
        cerr << "Too few arguments!" << endl
             << "Usage: ./manyToMany <inSgrGraph> <inNodes> <outMatrix>" << endl;
        exit(-1);
    }

    const string inGraphFn   = argv[1];
    const string inNodesFn   = argv[2];
    const string outMatrixFn = argv[3];

    const bool writeSourceNodes = false;
    const bool writeMatrix = false;
    const bool validateResult = false;

    VERBOSE( cerr << "read graph from '" << inGraphFn << "'" << endl);
    VERBOSE( cerr << "read nodes from '" << inNodesFn << "'" << endl);
    VERBOSE( cerr << "write matrix to '" << outMatrixFn << "'" << endl);
    VERBOSE( if (writeSourceNodes) cerr << "write source nodes to cerr" << endl );
    VERBOSE( if (writeMatrix)      cerr << "write matrix to cerr" << endl );
    VERBOSE( if (validateResult)   cerr << "validate result" << endl );

    // read graph
    MyGraph *const graph = readGraph(inGraphFn);

    VERBOSE( cerr << "Graph read." << endl );

    // read the input
    protos::ch::Nodes nodes;
    readMessage(inNodesFn, nodes);

    vector<NodeID> sources;
    readNodes(graph, nodes.sources(), sources);

    vector<NodeID> targets;
    readNodes(graph, nodes.targets(), targets);

    // create many-to-many object
    LevelID earlyStopLevel = 10;
    ManyToMany<MyGraph, DijkstraManyToManyFW, DijkstraManyToManyBW, performBucketScans> mtm(graph, earlyStopLevel);
{'fraction_non_alphanumeric': 0.12802011588498963, 'fraction_numerical': 0.0069968295616048976, 'mean_word_length': 3.767066180302241, 'pattern_counts': {'":': 0, '<': 160, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2181908', 'n_tokens_mistral': 2918, 'n_tokens_neox': 2812, 'n_words': 883}