<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import json
import os
import urllib2
from bgpranking import tools
from csv import DictWriter


import argparse
import glob
import redis


csv_dir = os.path.join('..', '..', 'website', 'data', 'csv')
agg_csv_dir = os.path.join('..', '..','website', 'data', 'csv_agg')
js_dir = os.path.join('..', '..', 'website', 'data', 'js')

ripe_url = 'https://stat.ripe.net/data/country-resource-list/data.json?resource={cc}'

def get_announces(ripe_url):
    handle = urllib2.urlopen(ripe_url, timeout=10)
    json_dump = handle.read()
    data = json.loads(json_dump)
    asns = data['data']['resources']['asn']
    return asns

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
            description='Bunch of function to create aggregations')
    parser.add_argument('-pka', '--push_known_asns', action='store_true',
            default=False)
    parser.add_argument('-cc', '--country_codes', nargs='+', type=str)
    parser.add_argument('-dcc', '--dump_country_codes', nargs='+', type=str)
    parser.add_argument('-map', '--make_map', action='store_true', default=False)
    args = parser.parse_args()
    r = redis.Redis(unix_socket_path='./redis_export.sock')

    if args.push_known_asns:
        csv_files = glob.glob(os.path.join(csv_dir, '*'))
        p = r.pipeline(False)
        [p.sadd('known_asns', csv_file) for csv_file in csv_files]
        p.execute()
        print 'Number of asns:', len(csv_files)
    elif args.country_codes is not None and len(args.country_codes) > 0:
        p = r.pipeline(False)
        p.sadd('countries', *args.country_codes)
        for cc in args.country_codes:
            url = ripe_url.format(cc=cc)
            asns = get_announces(url)
            if len(asns) > 0:
                p.sadd(cc, *asns)<commit_msg>add fallback mechanism in case the RIPE API is buggy.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import json
import os
import urllib2
from bgpranking import tools
from csv import DictWriter
import datetime

import argparse
import glob
import redis


csv_dir = os.path.join('..', '..', 'website', 'data', 'csv')
agg_csv_dir = os.path.join('..', '..','website', 'data', 'csv_agg')
js_dir = os.path.join('..', '..', 'website', 'data', 'js')

ripe_url = 'https://stat.ripe.net/data/country-resource-list/data.json?resource={cc}&time={day}'

def get_announces(ripe_url):
    handle = urllib2.urlopen(ripe_url, timeout=10)
    json_dump = handle.read()
    data = json.loads(json_dump)
    asns = data['data']['resources']['asn']
    return asns

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
            description='Bunch of function to create aggregations')
    parser.add_argument('-pka', '--push_known_asns', action='store_true',
            default=False)
    parser.add_argument('-cc', '--country_codes', nargs='+', type=str)
    parser.add_argument('-dcc', '--dump_country_codes', nargs='+', type=str)
    parser.add_argument('-map', '--make_map', action='store_true', default=False)
    args = parser.parse_args()
    r = redis.Redis(unix_socket_path='./redis_export.sock')

    if args.push_known_asns:
        csv_files = glob.glob(os.path.join(csv_dir, '*'))
        p = r.pipeline(False)
        [p.sadd('known_asns', csv_file) for csv_file in csv_files]
        p.execute()
        print 'Number of asns:', len(csv_files)
    elif args.country_codes is not None and len(args.country_codes) > 0:
        p = r.pipeline(False)
        p.sadd('countries', *args.country_codes)
        for cc in args.country_codes:
            date = datetime.date.today()
            counter = 0
            while True:
                date = date - datetime.timedelta(days=counter)
                url = ripe_url.format(cc=cc, day=date.isoformat())
                asns = get_announces(url)
                if len(asns) < 5:
                    counter += 1
                    continue
                p.sadd(cc, *asns)
                break
{'fraction_non_alphanumeric': 0.15756513026052105, 'fraction_numerical': 0.004008016032064128, 'mean_word_length': 2.914705882352941, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5818936', 'n_tokens_mistral': 1402, 'n_tokens_neox': 1354, 'n_words': 278}