<commit_before>import csv
import glob
import json
from math import acos, sin, cos, pi
import os
import re
import sys
import dateutil.parser

# Requires: probe information ("meta-probes.json")
# Requires: dateutil library (python3-dateutil in Debian & derivatives)
# Requires: curl
# Requires: IATA ("airports.dat")
# Requires: UN/LOCODE csv data ("2014-2 UNLOCODE CodeListPart[123].csv")

##
## Command-line parsing (hand parsing because I'm scared of argument libraries)
##
def syntax(argv):
    print("Syntax: {} measurement_id [date_start [date_end]]".format(argv[0]),
          file=sys.stderr)
    print("    measurement_id  - RIPE Atlas measurement ID",
          file=sys.stderr)
    print("    date_start      - date/time to start output at",
          file=sys.stderr)
    print("    date_end        - date/time to stop output at",
          file=sys.stderr)
    sys.exit(1)

if not (1 < len(sys.argv) <5):
    syntax(sys.argv)
    
measurement_id = int(sys.argv[1])

date_start_sec = None
date_end_sec = None
if len(sys.argv) > 2:
    date_start = dateutil.parser.parse(sys.argv[2])
    date_start_sec = int(date_start.timestamp())
if len(sys.argv) > 3:
    date_end = dateutil.parser.parse(sys.argv[3])
    date_end_sec = int(date_end.timestamp())

##
## Read in our probe information
##
probe_id_to_json = {}
with open('meta-probes.json') as fp:
    for line in fp:
        line = line.strip()
        if line:
            probe_json = json.loads(line)
            probe_id_to_json[probe_json["id"]] = probe_json

##
## Read in IATA locations
##
iata_city_to_lat_lon = {}
with open('airports.dat') as fp:
    airports = csv.reader(fp)
    for info in airports:
        iata_city = info[4]
        lat = float(info[6])
        lon = float(info[7])
        iata_city_to_lat_lon[iata_city] = (lat, lon)
    # special case LHR because we really want LCY
    iata_city_to_lat_lon["LHR"] = iata_city_to_lat_lon["LCY"]

## 
## Read in UN/LOCODE locations
##
un_locode_to_lat_lon = {}
for fname in glob.glob('*UNLOCODE*.csv'):
    with open(fname, encoding='latin_1') as fp:
        un_locodes = csv.reader(fp)
        for info in un_locodes:
            country = info[1]
            location = info[2]
            if info[10]:
                lat_str = info[10][0:5]
                lat = float(lat_str[:-1]) / 100.0
                if lat_str[-1] == 'S':
                    lat = -lat
                lon_str = info[10][6:]
                lon = float(lon_str[:-1]) / 100.0
                if lon_str[-1] == 'W':
                    lon = -lon
            else:
                lat = lon = None
            un_locode_to_lat_lon[country+location] = (lat, lon)
    # add a few sites since they're not properly listed
    un_locode_to_lat_lon["USIAD"] = iata_city_to_lat_lon["IAD"]
    un_locode_to_lat_lon["USLAX"] = iata_city_to_lat_lon["LAX"]
    un_locode_to_lat_lon["USNBN"] = iata_city_to_lat_lon["LGA"]
    un_locode_to_lat_lon["USSNN"] = iata_city_to_lat_lon["SJC"]


##
## mapping from HOSTNAME.BIND string to identifier, latitude, and longitude
##
def node_name_to_city_lat_lon(node_name):
    city = None

    # this is the CloudFlare format - so simple! :)
    if len(node_name) == 3:
        city = node_name.upper()
        lat, lon = iata_city_to_lat_lon[city]

    # this is the CIRA ANY.CA-SERVERS.CA mapping of HOSTNAME.BIND
    m = re.search(r'^ns\d\d.([a-z]{3}).(ca-servers|d-zone).ca$',
                  node_name, re.I)
    if m:
        city = m.group(1).upper()
        # special case MTL because the Canadians really mean YUL
        if city == "MTL":
            city = "YUL"
        lat, lon = iata_city_to_lat_lon[city]

    # this is the Dyn Hivecast mapping of HOSTNAME.BIND
    m = re.search(r'^hivecast-\d\d-([a-z]{2})([a-z]{3}).as15135.net \S+$',
                  node_name)
    if m:
        country = m.group(1).upper()
        loc = m.group(2).upper()
        city = country+loc
        lat, lon = un_locode_to_lat_lon[city]

    if city:
        return city, lat, lon
    else:
        return None

## 
## Download the measurement JSON
##
if date_end_sec is not None:
    url = ("https://atlas.ripe.net/api/v1/measurement/{}/result/?"
           "start={}&stop={}&format=json".format(measurement_id,
                                                 date_start_sec, date_end_sec))
    meas_fname = "atlas-meas-{}-{}-{}".format(measurement_id,
                                              date_start_sec, date_end_sec)
elif date_start_sec is not None:
    url = ("https://atlas.ripe.net/api/v1/measurement/{}/result/?"
           "start={}&format=json".format(measurement_id, date_start_sec))
    meas_fname = "atlas-meas-{}-{}".format(measurement_id, date_start_sec)
else:
    url = ("https://atlas.ripe.net/api/v1/measurement/{}/result/?"
           "format=json".format(measurement_id))
    meas_fname = "atlas-meas-{}".format(measurement_id)

try:
    fp = open(meas_fname + ".json")
except FileNotFoundError:
    os.system("curl --output '{}' --url '{}'".format(meas_fname + ".json", url))
    try:
        fp = open(meas_fname + ".json")
    except FileNotFoundError:
        print("Unable to get '{}'".format(url), file=sys.stderr)
        sys.exit(1)



##
## Add probe info into the measurements, also get instance sites
##
cnt_total = 0
cnt_good = 0
measurements = []          # our updated set of measurements
missing_probes = set()     # probes that are missing from our probe dataset
instance_to_lat_lon = {}   # sites with anycast instances
for measurement in json.load(fp):
    cnt_total += 1
    fixed_json = measurement.copy()
    prb_id = measurement["prb_id"]
    if prb_id not in probe_id_to_json:<commit_msg>Use human-readable date/time in file names<commit_after>import csv
import glob
import json
from math import acos, sin, cos, pi
import os
import re
import sys
import time

import dateutil.parser

# Requires: probe information ("meta-probes.json")
# Requires: dateutil library (python3-dateutil in Debian & derivatives)
# Requires: curl
# Requires: IATA ("airports.dat")
# Requires: UN/LOCODE csv data ("2014-2 UNLOCODE CodeListPart[123].csv")

##
## Command-line parsing (hand parsing because I'm scared of argument libraries)
##
def syntax(argv):
    print("Syntax: {} measurement_id [date_start [date_end]]".format(argv[0]),
          file=sys.stderr)
    print("    measurement_id  - RIPE Atlas measurement ID",
          file=sys.stderr)
    print("    date_start      - date/time to start output at",
          file=sys.stderr)
    print("    date_end        - date/time to stop output at",
          file=sys.stderr)
    sys.exit(1)

if not (1 < len(sys.argv) <5):
    syntax(sys.argv)
    
measurement_id = int(sys.argv[1])

date_start_sec = None
date_end_sec = None
if len(sys.argv) > 2:
    date_start = dateutil.parser.parse(sys.argv[2])
    date_start_sec = int(date_start.timestamp())
    date_start_iso = time.strftime("%Y%m%dT%H%M%S", time.gmtime(date_start_sec))
if len(sys.argv) > 3:
    date_end = dateutil.parser.parse(sys.argv[3])
    date_end_sec = int(date_end.timestamp())
    date_end_iso = time.strftime("%Y%m%dT%H%M%S", time.gmtime(date_end_sec))

##
## Read in our probe information
##
probe_id_to_json = {}
with open('meta-probes.json') as fp:
    for line in fp:
        line = line.strip()
        if line:
            probe_json = json.loads(line)
            probe_id_to_json[probe_json["id"]] = probe_json

##
## Read in IATA locations
##
iata_city_to_lat_lon = {}
with open('airports.dat') as fp:
    airports = csv.reader(fp)
    for info in airports:
        iata_city = info[4]
        lat = float(info[6])
        lon = float(info[7])
        iata_city_to_lat_lon[iata_city] = (lat, lon)
    # special case LHR because we really want LCY
    iata_city_to_lat_lon["LHR"] = iata_city_to_lat_lon["LCY"]

## 
## Read in UN/LOCODE locations
##
un_locode_to_lat_lon = {}
for fname in glob.glob('*UNLOCODE*.csv'):
    with open(fname, encoding='latin_1') as fp:
        un_locodes = csv.reader(fp)
        for info in un_locodes:
            country = info[1]
            location = info[2]
            if info[10]:
                lat_str = info[10][0:5]
                lat = float(lat_str[:-1]) / 100.0
                if lat_str[-1] == 'S':
                    lat = -lat
                lon_str = info[10][6:]
                lon = float(lon_str[:-1]) / 100.0
                if lon_str[-1] == 'W':
                    lon = -lon
            else:
                lat = lon = None
            un_locode_to_lat_lon[country+location] = (lat, lon)
    # add a few sites since they're not properly listed
    un_locode_to_lat_lon["USIAD"] = iata_city_to_lat_lon["IAD"]
    un_locode_to_lat_lon["USLAX"] = iata_city_to_lat_lon["LAX"]
    un_locode_to_lat_lon["USNBN"] = iata_city_to_lat_lon["LGA"]
    un_locode_to_lat_lon["USSNN"] = iata_city_to_lat_lon["SJC"]


##
## mapping from HOSTNAME.BIND string to identifier, latitude, and longitude
##
def node_name_to_city_lat_lon(node_name):
    city = None

    # this is the CloudFlare format - so simple! :)
    if len(node_name) == 3:
        city = node_name.upper()
        lat, lon = iata_city_to_lat_lon[city]

    # this is the CIRA ANY.CA-SERVERS.CA mapping of HOSTNAME.BIND
    m = re.search(r'^ns\d\d.([a-z]{3}).(ca-servers|d-zone).ca$',
                  node_name, re.I)
    if m:
        city = m.group(1).upper()
        # special case MTL because the Canadians really mean YUL
        if city == "MTL":
            city = "YUL"
        lat, lon = iata_city_to_lat_lon[city]

    # this is the Dyn Hivecast mapping of HOSTNAME.BIND
    m = re.search(r'^hivecast-\d\d-([a-z]{2})([a-z]{3}).as15135.net \S+$',
                  node_name)
    if m:
        country = m.group(1).upper()
        loc = m.group(2).upper()
        city = country+loc
        lat, lon = un_locode_to_lat_lon[city]

    if city:
        return city, lat, lon
    else:
        return None

## 
## Download the measurement JSON
##
if date_end_sec is not None:
    url = ("https://atlas.ripe.net/api/v1/measurement/{}/result/?"
           "start={}&stop={}&format=json".format(measurement_id,
                                                 date_start_sec, date_end_sec))
    meas_fname = "atlas-meas-{}-{}-{}".format(measurement_id,
                                              date_start_iso, date_end_iso)
elif date_start_sec is not None:
    url = ("https://atlas.ripe.net/api/v1/measurement/{}/result/?"
           "start={}&format=json".format(measurement_id, date_start_sec))
    meas_fname = "atlas-meas-{}-{}".format(measurement_id, date_start_iso)
else:
    url = ("https://atlas.ripe.net/api/v1/measurement/{}/result/?"
           "format=json".format(measurement_id))
    meas_fname = "atlas-meas-{}".format(measurement_id)

try:
    fp = open(meas_fname + ".json")
except FileNotFoundError:
    os.system("curl --output '{}' --url '{}'".format(meas_fname + ".json", url))
    try:
        fp = open(meas_fname + ".json")
    except FileNotFoundError:
        print("Unable to get '{}'".format(url), file=sys.stderr)
        sys.exit(1)



##
## Add probe info into the measurements, also get instance sites
##
cnt_total = 0
cnt_good = 0
measurements = []          # our updated set of measurements
missing_probes = set()     # probes that are missing from our probe dataset
instance_to_lat_lon = {}   # sites with anycast instances
for measurement in json.load(fp):
    cnt_total += 1
    fixed_json = measurement.copy()
    prb_id = measurement["prb_id"]
    if prb_id not in probe_id_to_json:
{'fraction_non_alphanumeric': 0.1346153846153846, 'fraction_numerical': 0.011088011088011088, 'mean_word_length': 2.5954531298660855, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7115929', 'n_tokens_mistral': 4123, 'n_tokens_neox': 3981, 'n_words': 892}