<commit_before>import argparse
import json
import random

from colorama import Fore, Style
from shapely import wkb
import sqlite3

from lib import logger
from lib.utils import pairwise
from pfds import pfds
from visualize import visualize


LOG = logger.getLogger()


def connect_to_db(db_name):
    db = sqlite3.connect(db_name, isolation_level=None)
    db.enable_load_extension(True)
    db.load_extension('libspatialite')
    return db.cursor()


def load_graph(cur):
    # Graph definition
    G = {}  # Graph
    G_reversed = {}  # Graph with reversed edges - for A* landmarks heuristic
    P = {}  # Geographical points
    L = {}  # Roads geometries,

    # Load graph vertices
    node_qr = "SELECT node_id, AsBinary(geometry) AS point FROM roads_nodes;"
    cur.execute(node_qr)
    nodes = cur.fetchall()
    for i, geometry in nodes:
        G_reversed[i] = {}
        G[i] = {}
        L[i] = {}
        P[i] = wkb.loads(str(geometry))

    # Load graph edges
    road_qr = ("SELECT node_from, node_to, oneway_fromto, oneway_tofrom, "
               "length, AsBinary(geometry) as line FROM roads;")
    cur.execute(road_qr)
    roads = cur.fetchall()
    for node_from, node_to, fromto, tofrom, length, geometry in roads:
        if fromto:
            G[node_from][node_to] = length
            G_reversed[node_to][node_from] = length
            L[node_from][node_to] = wkb.loads(str(geometry))
        if tofrom:
            G[node_to][node_from] = length
            G_reversed[node_from][node_to] = length
            L[node_to][node_from] = wkb.loads(str(geometry))

    return G, G_reversed, P, L


def get_pairs(G, filename, rand_num):
    pairs = []

    if filename:
        with open(filename) as f:
            json_pairs = json.load(f)

        for pair in json_pairs:
            pairs.append((pair['src'], pair['dest']))

    while len(pairs) < rand_num:
        src = random.choice(G.keys())
        dest = random.choice(G.keys())
        if src != dest:
            pairs.append((src, dest))

    return pairs


def baseline_query(G, L, P, pairs, pfd):
    baseline = {}

    for src, dest in pairs:
        path, visited = pfd.calc(src, dest)

        cost = 0.0
        for i in range(1, len(path)):
            cost += G[path[i - 1]][path[i]]

        baseline['%d-%d' % (src, dest)] = len(visited), cost

    return baseline


def query(G, L, P, pairs, pfd, runs, baseline):
    results = {}

    for i in xrange(runs):
        results[i] = {}
        for src, dest in pairs:
            path, visited = pfd.calc(src, dest)

            cost = 0.0
            for j in range(1, len(path)):
                cost += G[path[j - 1]][path[j]]

            if cost != baseline['%d-%d' % (src, dest)][1]:
                LOG.error(Fore.RED + "Paths doesn't match!" + Fore.RESET)

            p = (float(len(visited)) /
                 float(baseline['%d-%d' % (src, dest)][0])) * 100

            results[i]['%d-%d' % (src, dest)] = p

        if runs > 1 and i != runs - 1:
            pfd.calculate_landmarks()

    # We need to flatten the results using average
    # TODO: Calculating also standard deviation here would be beneficial.
    flat_results = {}

    for src, dest in pairs:
        k = '%d-%d' % (src, dest)
        avg = 0.
        for i in xrange(runs):
            avg += results[i][k]

        flat_results[k] = avg / runs

    return results


def main():
    parser = argparse.ArgumentParser(description='ALT-Tester')

    parser.add_argument('--db', action="store", default='gdansk_cleaned.sqlite')
    parser.add_argument('--landmarks', action="store", type=int, default=16)
    parser.add_argument('--random-pairs', action="store", type=int, default=50)
    parser.add_argument('--pairs-file', action="store")

    arguments = parser.parse_args()

    db_name = arguments.db
    lm_num = arguments.landmarks<commit_msg>Fixing a bug...<commit_after>import argparse
import json
import random

from colorama import Fore, Style
from shapely import wkb
import sqlite3

from lib import logger
from lib.utils import pairwise
from pfds import pfds
from visualize import visualize


LOG = logger.getLogger()


def connect_to_db(db_name):
    db = sqlite3.connect(db_name, isolation_level=None)
    db.enable_load_extension(True)
    db.load_extension('libspatialite')
    return db.cursor()


def load_graph(cur):
    # Graph definition
    G = {}  # Graph
    G_reversed = {}  # Graph with reversed edges - for A* landmarks heuristic
    P = {}  # Geographical points
    L = {}  # Roads geometries,

    # Load graph vertices
    node_qr = "SELECT node_id, AsBinary(geometry) AS point FROM roads_nodes;"
    cur.execute(node_qr)
    nodes = cur.fetchall()
    for i, geometry in nodes:
        G_reversed[i] = {}
        G[i] = {}
        L[i] = {}
        P[i] = wkb.loads(str(geometry))

    # Load graph edges
    road_qr = ("SELECT node_from, node_to, oneway_fromto, oneway_tofrom, "
               "length, AsBinary(geometry) as line FROM roads;")
    cur.execute(road_qr)
    roads = cur.fetchall()
    for node_from, node_to, fromto, tofrom, length, geometry in roads:
        if fromto:
            G[node_from][node_to] = length
            G_reversed[node_to][node_from] = length
            L[node_from][node_to] = wkb.loads(str(geometry))
        if tofrom:
            G[node_to][node_from] = length
            G_reversed[node_from][node_to] = length
            L[node_to][node_from] = wkb.loads(str(geometry))

    return G, G_reversed, P, L


def get_pairs(G, filename, rand_num):
    pairs = []

    if filename:
        with open(filename) as f:
            json_pairs = json.load(f)

        for pair in json_pairs:
            pairs.append((pair['src'], pair['dest']))

    while len(pairs) < rand_num:
        src = random.choice(G.keys())
        dest = random.choice(G.keys())
        if src != dest:
            pairs.append((src, dest))

    return pairs


def baseline_query(G, L, P, pairs, pfd):
    baseline = {}

    for src, dest in pairs:
        path, visited = pfd.calc(src, dest)

        cost = 0.0
        for i in range(1, len(path)):
            cost += G[path[i - 1]][path[i]]

        baseline['%d-%d' % (src, dest)] = len(visited), cost

    return baseline


def query(G, L, P, pairs, pfd, runs, baseline):
    results = {}

    for i in xrange(runs):
        results[i] = {}
        for src, dest in pairs:
            path, visited = pfd.calc(src, dest)

            cost = 0.0
            for j in range(1, len(path)):
                cost += G[path[j - 1]][path[j]]

            if cost != baseline['%d-%d' % (src, dest)][1]:
                LOG.error(Fore.RED + "Paths doesn't match!" + Fore.RESET)

            p = (float(len(visited)) /
                 float(baseline['%d-%d' % (src, dest)][0])) * 100

            results[i]['%d-%d' % (src, dest)] = p

        if runs > 1 and i != runs - 1:
            pfd.calculate_landmarks()

    # We need to flatten the results using average
    # TODO: Calculating also standard deviation here would be beneficial.
    flat_results = {}

    for src, dest in pairs:
        k = '%d-%d' % (src, dest)
        avg = 0.
        for i in xrange(runs):
            avg += results[i][k]

        flat_results[k] = avg / runs

    return flat_results


def main():
    parser = argparse.ArgumentParser(description='ALT-Tester')

    parser.add_argument('--db', action="store", default='gdansk_cleaned.sqlite')
    parser.add_argument('--landmarks', action="store", type=int, default=16)
    parser.add_argument('--random-pairs', action="store", type=int, default=50)
    parser.add_argument('--pairs-file', action="store")

    arguments = parser.parse_args()

    db_name = arguments.db
    lm_num = arguments.landmarks
{'fraction_non_alphanumeric': 0.13143299367986586, 'fraction_numerical': 0.005675222494518251, 'mean_word_length': 2.388986013986014, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2412445', 'n_tokens_mistral': 2633, 'n_tokens_neox': 2498, 'n_words': 655}