<commit_before>import re
import random
import hashlib
import requests
import tempfile
import time
import os
import argparse
try:
    from mar import MarFile
except ImportError:
    from mar.mar import MarFile


partial_mar_re = re.compile(
    r"update/(?P<platform>[\w\d\-]+)/(?P<locale>\w+)/"
    "(?P<product>\w+)-(?P<from>[\d\.]+)-(?P<to>[\d\.]+)\.partial\.mar$")
complete_mar_re = re.compile(
    r"update/(?P<platform>[\w\d\-]+)/(?P<locale>\w+)/"
    "(?P<product>\w+)-(?P<version>[\d\.]+)\.complete\.mar$")

FTP_URL = "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/{v}"


def mar_internal_sha512(filename):
    m = MarFile(filename)
    digest = hashlib.new("sha512")
    for member in m.members:
        m.fileobj.seek(member._offset)
        digest.update(m.fileobj.read(member.size))
    return str(digest.hexdigest())


def build_update_tree(from_release):
    completes = {}
    partials = {}
    base_url = FTP_URL.format(v=from_release)
    sha512_url = "{}/SHA512SUMS".format(base_url)
    text = requests.get(sha512_url).text
    for line in text.splitlines():
        sha, filename = line.split(None, 1)
        fileurl = "{}/{}".format(base_url, filename)
        m_p = partial_mar_re.match(filename)
        if m_p:
            platform = m_p.group("platform")
            locale = m_p.group("locale")
            product = m_p.group("product")
            from_version = m_p.group("from")
            to_version = m_p.group("to")
            partials[(platform, locale, product, from_version,
                      to_version)] = (fileurl, sha)

        m_c = complete_mar_re.match(filename)
        if m_c:
            platform = m_c.group("platform")
            locale = m_c.group("locale")
            product = m_c.group("product")
            version = m_c.group("version")
            completes[(platform, locale, product, version)] = (fileurl, sha)
    return (completes, partials)


def trigger_partial(api_root, data):
    rv = requests.post("{}/partial".format(api_root), data=data)
    rv.raise_for_status()


def get_file(from_, to, attempt=0, max_attempts=100, interval=5):
    print "Downloading", from_, "to", to
    while True:
        r = requests.get(from_)
        r.raise_for_status()
        if r.status_code == 202:
            attempt += 1
            if attempt >= max_attempts:
                raise Exception("Timed out")
            print ".",
            time.sleep(interval)
        elif r.status_code == 200:
            with open(to, mode="wb") as f:
                f.write(r.content)
            break
        else:
            raise Exception("Unexpected status code: %s", r.status_code)


def build_sample(completes, partials, channel_id):
    sample = []
    for k, v in partials.iteritems():
        try:
            platform, locale, product, from_version, to_version = k
            fileurl, sha = v
            mar_from, sha_from = completes[(platform, locale, product,
                                            from_version)]
            mar_to, sha_to = completes[(platform, locale, product, to_version)]
            data = dict(mar_from=mar_from, mar_to=mar_to, sha_from=sha_from,
                        sha_to=sha_to, channel_id=channel_id,
                        product_version=to_version)
            identifier = "{}-{}".format(sha_from, sha_to)
            sample.append((data, fileurl, identifier))
        except KeyError:
            pass
    return sample


def compare_partials(args):
    data, mar_url, api_root, identifier = args
    trigger_partial(api_root, data)
    _, their_mar = tempfile.mkstemp()<commit_msg>Retry on HTTP 502<commit_after>import re
import random
import hashlib
import requests
import tempfile
import time
import os
import argparse
try:
    from mar import MarFile
except ImportError:
    from mar.mar import MarFile


partial_mar_re = re.compile(
    r"update/(?P<platform>[\w\d\-]+)/(?P<locale>\w+)/"
    "(?P<product>\w+)-(?P<from>[\d\.]+)-(?P<to>[\d\.]+)\.partial\.mar$")
complete_mar_re = re.compile(
    r"update/(?P<platform>[\w\d\-]+)/(?P<locale>\w+)/"
    "(?P<product>\w+)-(?P<version>[\d\.]+)\.complete\.mar$")

FTP_URL = "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/{v}"


def mar_internal_sha512(filename):
    m = MarFile(filename)
    digest = hashlib.new("sha512")
    for member in m.members:
        m.fileobj.seek(member._offset)
        digest.update(m.fileobj.read(member.size))
    return str(digest.hexdigest())


def build_update_tree(from_release):
    completes = {}
    partials = {}
    base_url = FTP_URL.format(v=from_release)
    sha512_url = "{}/SHA512SUMS".format(base_url)
    text = requests.get(sha512_url).text
    for line in text.splitlines():
        sha, filename = line.split(None, 1)
        fileurl = "{}/{}".format(base_url, filename)
        m_p = partial_mar_re.match(filename)
        if m_p:
            platform = m_p.group("platform")
            locale = m_p.group("locale")
            product = m_p.group("product")
            from_version = m_p.group("from")
            to_version = m_p.group("to")
            partials[(platform, locale, product, from_version,
                      to_version)] = (fileurl, sha)

        m_c = complete_mar_re.match(filename)
        if m_c:
            platform = m_c.group("platform")
            locale = m_c.group("locale")
            product = m_c.group("product")
            version = m_c.group("version")
            completes[(platform, locale, product, version)] = (fileurl, sha)
    return (completes, partials)


def trigger_partial(api_root, data):
    attempt = 0
    max_attempts = 100
    interval = 5
    while True:
        r = requests.post("{}/partial".format(api_root), data=data)
        # Retry on 502
        if r.status_code == 502:
            attempt += 1
            if attempt >= max_attempts:
                raise Exception("Timed out")
            print ".",
            time.sleep(interval)
        elif str(r.status_code).startswith("2"):
            break
        else:
            raise Exception("Unexpected status code: %s, %s", r.status_code,
                            data)


def get_file(from_, to, attempt=0, max_attempts=100, interval=5):
    print "Downloading", from_, "to", to
    while True:
        r = requests.get(from_)
        # Retry on 502 too
        if r.status_code in (202, 502):
            attempt += 1
            if attempt >= max_attempts:
                raise Exception("Timed out")
            print ".",
            time.sleep(interval)
        elif r.status_code == 200:
            with open(to, mode="wb") as f:
                f.write(r.content)
            break
        else:
            raise Exception("Unexpected status code: %s %s", r.status_code,
                            from_)


def build_sample(completes, partials, channel_id):
    sample = []
    for k, v in partials.iteritems():
        try:
            platform, locale, product, from_version, to_version = k
            fileurl, sha = v
            mar_from, sha_from = completes[(platform, locale, product,
                                            from_version)]
            mar_to, sha_to = completes[(platform, locale, product, to_version)]
            data = dict(mar_from=mar_from, mar_to=mar_to, sha_from=sha_from,
                        sha_to=sha_to, channel_id=channel_id,
                        product_version=to_version)
            identifier = "{}-{}".format(sha_from, sha_to)
            sample.append((data, fileurl, identifier))
        except KeyError:
            pass
    return sample


def compare_partials(args):
    data, mar_url, api_root, identifier = args
    trigger_partial(api_root, data)
    _, their_mar = tempfile.mkstemp()
{'fraction_non_alphanumeric': 0.13309166235111342, 'fraction_numerical': 0.01009839461418954, 'mean_word_length': 2.2241235392320533, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3069783', 'n_tokens_mistral': 2515, 'n_tokens_neox': 2385, 'n_words': 500}