<commit_before>#!/usr/bin/env python

import json
import csv

as_rel = {}

def rel_char( src, dst ):
    rv = '?'
    key = "{0}+{1}".format(src, dst)
    key_rev = "{0}+{1}".format(dst, src)
    if as_rel.has_key(key):
        rv = '-' if (as_rel[key] == 0) else '>'
    elif as_rel.has_key(key_rev):
        rv = '-' if (as_rel[key_rev] == 0) else '<'

    return rv

with open('data/rv-nz-aspath.json', 'rb') as rv_file:
    rv_paths = json.load(rv_file)

with open('as-rank/20140601.as-rel.txt', 'rb') as as_rel_file:
    as_rel_csv = csv.reader(filter(lambda row: row[0]!='#',as_rel_file), delimiter="|")
    for as_rel_entry in as_rel_csv:
        [ prov_as, cust_as, rel ] = as_rel_entry
        as_rel["{0}+{1}".format(prov_as,cust_as)] = int(rel)


# print "{} {} {}".format('9500', rel_char('9500', '9901'), '9901')

unknown = []
for aspath in rv_paths['aspath']:
    if len(aspath) > 1:
        deltas = [ aspath[0] ]
        for i in range(1, len(aspath)):
            rel = rel_char( aspath[i-1], aspath[i])
            deltas.append( rel )
            deltas.append(aspath[i])
            if rel == '?':
                unknown.append( "|".join([aspath[i-1], aspath[i], rel]))
        if (len(deltas) > 0):
#            print " ".join(deltas)
            pass
        else:
            print "WTF with {}".format(aspath)

with open('unknown-rel.csv', 'w') as unk_rel_file:
    unk_out = csv.writer(unk_rel_file, lineterminator="\n")

    for unk in set(unknown):
        unk_out.writerow([unk])
<commit_msg>Some code refactoring. Relation symbol accounts now 3 different values on the relation parameter<commit_after>#!/usr/bin/env python

import json
import csv

as_rel = {}

def rel_char( src, dst ):
    rv = '?'
    key = "{0}+{1}".format(src, dst)
    key_rev = "{0}+{1}".format(dst, src)
    if as_rel.has_key(key):
        if (as_rel[key] == 0):
            rv = '-'
        elif (as_rel[key] < 0):
            rv = '>'
        else:
            rv = '<'
    elif as_rel.has_key(key_rev):
        if (as_rel[key_rev] == 0):
            rv = '-'
        elif (as_rel[key_rev] < 0):
            rv = '<'
        else:
            rv = '>'

    return rv

def load_as_relationship(file_list):
    as_rel = {}
    for filename in file_list:
        with open(filename, 'r') as as_rel_file:
            as_rel_csv = csv.reader(filter(lambda row: row[0]!='#',as_rel_file), delimiter="|")
            for as_rel_entry in as_rel_csv:
                [ prov_as, cust_as, rel ] = as_rel_entry
                as_rel["{0}+{1}".format(prov_as,cust_as)] = int(rel)

    return as_rel

with open('data/rv-nz-aspath.json', 'rb') as rv_file:
    rv_paths = json.load(rv_file)

as_rel = load_as_relationship(['as-rank/20140601.as-rel.txt', 'data/local-as-rel-info.csv'])

# print "{} {} {}".format('9500', rel_char('9500', '9901'), '9901')

unknown = []
for aspath in rv_paths['aspath']:
    if len(aspath) > 1:
        deltas = [ aspath[0] ]
        for i in range(1, len(aspath)):
            rel = rel_char( aspath[i-1], aspath[i])
            deltas.append( rel )
            deltas.append(aspath[i])
            if rel == '?':
                unknown.append( "|".join([aspath[i-1], aspath[i], rel]))
        if (len(deltas) > 0):
            print " ".join(deltas)
            pass
        else:
            print "WTF with {}".format(aspath)

with open('unknown-rel.csv', 'w') as unk_rel_file:
    unk_out = csv.writer(unk_rel_file, lineterminator="\n")

    for unk in set(unknown):
        unk_out.writerow([unk])

{'fraction_non_alphanumeric': 0.17413499716392514, 'fraction_numerical': 0.02297220646625071, 'mean_word_length': 2.352661596958175, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4078730', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1330, 'n_words': 271}