<commit_before>###
# Utilities for talking to esmond


import pscheduler
import requests
import socket

#disable SSL warnings
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

log = pscheduler.Log(prefix="archiver-esmond", quiet=True)

DEFAULT_SUMMARIES = {
    "throughput": [
        {
            "summary-window":   86400,
            "event-type":   "throughput",
            "summary-type":   "average",
        },
    ],
    "packet-loss-rate": [
        {
            "summary-window":   300,
            "event-type":   "packet-loss-rate",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   3600,
            "event-type":   "packet-loss-rate",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "packet-loss-rate",
            "summary-type":   "aggregation",
        },
    ],
    "histogram-owdelay": [
        {
            "summary-window":   300,
            "event-type":   "histogram-owdelay",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   300,
            "event-type":   "histogram-owdelay",
            "summary-type":   "statistics",
        },
        {
            "summary-window":   3600,
            "event-type":   "histogram-owdelay",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   0,
            "event-type":   "histogram-owdelay",
            "summary-type":   "statistics",
        },
        {
            "summary-window":   3600,
            "event-type":   "histogram-owdelay",
            "summary-type":   "statistics",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-owdelay",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-owdelay",
            "summary-type":  "statistics",
        },
    ],
    "packet-loss-rate-bidir":[
        {
            "summary-window":   3600,
            "event-type":   "packet-loss-rate-bidir",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "packet-loss-rate-bidir",
            "summary-type":   "aggregation",
        },
    ],
    "histogram-rtt": [
        {
            "summary-window":   3600,
            "event-type":   "histogram-rtt",
            "summary-type":   "aggregation",
        },
    
        {
            "summary-window":   3600,
            "event-type":   "histogram-rtt",
            "summary-type":  "statistics",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-rtt",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-rtt",
            "summary-type": "statistics",
        }
    ],
}


###
# Utility functions
def iso8601_to_seconds(val):
    td = pscheduler.iso8601_as_timedelta(val)
    return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10.0**6) / 10.0**6

def get_ips(addr):
    ip_v4 = None
    ip_v6 = None
    try:
        addrinfo = socket.getaddrinfo(addr, None)
        for ai in addrinfo:
            if ai[0] == socket.AF_INET:
                ip_v4 = ai[4][0]
            elif ai[0] == socket.AF_INET6:
                ip_v6 = ai[4][0]
    except:
        pass
    return ip_v4, ip_v6
    
def normalize_ip_versions(src, dest, ip_version=None):
    src_ip = None
    dest_ip = None
    src_ip_v4, src_ip_v6 = get_ips(src)
    dest_ip_v4, dest_ip_v6 = get_ips(dest)
    #prefer v6 if not specified
    if not ip_version or ip_version == 6:
       if src_ip_v6 and dest_ip_v6:
            src_ip = src_ip_v6
            dest_ip = dest_ip_v6
    if not src_ip or not dest_ip or ip_version == 4:
        if src_ip_v4 and dest_ip_v4:
            src_ip = src_ip_v4
            dest_ip = dest_ip_v4

    return src_ip, dest_ip

def handle_storage_error(result, attempts=0, policy=[]):
    #build object
    retry = False
    archive_err_result = { 'succeeded': False, 'error': result }
    policy_attempt_sum = 0
    for p in policy:
        policy_attempt_sum += p['attempts']
        if policy_attempt_sum > attempts:
            retry = True
            archive_err_result['retry'] = p['wait']
            break
    if retry:
        pscheduler.succeed_json(archive_err_result)
    else:
        pscheduler.fail("Archiver permanently abandoned registering test after %d attempt(s): %s" % (attempts+1, result))


###
# Utility classes
class EsmondClient:
    url = ""
    verify_ssl = False<commit_msg>Properly setting exit status when archiver permanently gives-up on retries<commit_after>###
# Utilities for talking to esmond


import pscheduler
import requests
import socket

#disable SSL warnings
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

log = pscheduler.Log(prefix="archiver-esmond", quiet=True)

DEFAULT_SUMMARIES = {
    "throughput": [
        {
            "summary-window":   86400,
            "event-type":   "throughput",
            "summary-type":   "average",
        },
    ],
    "packet-loss-rate": [
        {
            "summary-window":   300,
            "event-type":   "packet-loss-rate",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   3600,
            "event-type":   "packet-loss-rate",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "packet-loss-rate",
            "summary-type":   "aggregation",
        },
    ],
    "histogram-owdelay": [
        {
            "summary-window":   300,
            "event-type":   "histogram-owdelay",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   300,
            "event-type":   "histogram-owdelay",
            "summary-type":   "statistics",
        },
        {
            "summary-window":   3600,
            "event-type":   "histogram-owdelay",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   0,
            "event-type":   "histogram-owdelay",
            "summary-type":   "statistics",
        },
        {
            "summary-window":   3600,
            "event-type":   "histogram-owdelay",
            "summary-type":   "statistics",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-owdelay",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-owdelay",
            "summary-type":  "statistics",
        },
    ],
    "packet-loss-rate-bidir":[
        {
            "summary-window":   3600,
            "event-type":   "packet-loss-rate-bidir",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "packet-loss-rate-bidir",
            "summary-type":   "aggregation",
        },
    ],
    "histogram-rtt": [
        {
            "summary-window":   3600,
            "event-type":   "histogram-rtt",
            "summary-type":   "aggregation",
        },
    
        {
            "summary-window":   3600,
            "event-type":   "histogram-rtt",
            "summary-type":  "statistics",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-rtt",
            "summary-type":   "aggregation",
        },
        {
            "summary-window":   86400,
            "event-type":   "histogram-rtt",
            "summary-type": "statistics",
        }
    ],
}


###
# Utility functions
def iso8601_to_seconds(val):
    td = pscheduler.iso8601_as_timedelta(val)
    return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10.0**6) / 10.0**6

def get_ips(addr):
    ip_v4 = None
    ip_v6 = None
    try:
        addrinfo = socket.getaddrinfo(addr, None)
        for ai in addrinfo:
            if ai[0] == socket.AF_INET:
                ip_v4 = ai[4][0]
            elif ai[0] == socket.AF_INET6:
                ip_v6 = ai[4][0]
    except:
        pass
    return ip_v4, ip_v6
    
def normalize_ip_versions(src, dest, ip_version=None):
    src_ip = None
    dest_ip = None
    src_ip_v4, src_ip_v6 = get_ips(src)
    dest_ip_v4, dest_ip_v6 = get_ips(dest)
    #prefer v6 if not specified
    if not ip_version or ip_version == 6:
       if src_ip_v6 and dest_ip_v6:
            src_ip = src_ip_v6
            dest_ip = dest_ip_v6
    if not src_ip or not dest_ip or ip_version == 4:
        if src_ip_v4 and dest_ip_v4:
            src_ip = src_ip_v4
            dest_ip = dest_ip_v4

    return src_ip, dest_ip

def handle_storage_error(result, attempts=0, policy=[]):
    #build object
    archive_err_result = { 'succeeded': False, 'error': result }
    policy_attempt_sum = 0
    for p in policy:
        policy_attempt_sum += p['attempts']
        if policy_attempt_sum > attempts:
            archive_err_result['retry'] = p['wait']
            break
    
    pscheduler.succeed_json(archive_err_result)

###
# Utility classes
class EsmondClient:
    url = ""
    verify_ssl = False
{'fraction_non_alphanumeric': 0.1299601091748898, 'fraction_numerical': 0.02592903632164602, 'mean_word_length': 1.65154467019204, 'pattern_counts': {'":': 112, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6953334', 'n_tokens_mistral': 3174, 'n_tokens_neox': 3027, 'n_words': 546}