<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License..

import os
import platform
import random
import re
import subprocess
import sys
import tempfile
import time
import urllib2
import contextlib

BOOT_STRAP_FN = os.path.join(os.path.expanduser("~"), ".anvil_strapped")
RH_EPEL_URLS = [
    'http://mirror.cogentco.com/pub/linux/epel/6/i386',
    "http://epel.mirror.freedomvoice.com/6/i386",
    'http://mirrors.kernel.org/fedora-epel/6/i386',
]
URL_TIMEOUT = 5


def tiny_p(cmd, capture=True):
    # Darn python 2.6 doesn't have check_output (argggg)
    stdout = subprocess.PIPE
    stderr = subprocess.PIPE
    if not capture:
        stdout = None
        stderr = None
    sp = subprocess.Popen(cmd, stdout=stdout,
                    stderr=stderr, stdin=None)
    (out, err) = sp.communicate()
    if sp.returncode not in [0]:
        raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" 
                            % (cmd, sp.returncode, out, err))
    return (out, err)


def _write_msg(msg):
    sys.stderr.write("%s\n" % (msg))


def _write_warn(msg):
    _write_msg("WARNING: %s" % (msg))


def _strap_epel():
    exceptions = []
    fetched = False
    _write_msg("Installing EPEL repositories...")
    for u in RH_EPEL_URLS:
        try:
            with contextlib.closing(urllib2.urlopen(u, timeout=URL_TIMEOUT)) as uh:
                rel = re.search("epel-release(.*?)[.]rpm", uh.read(), re.I)
            if rel:
                with tempfile.NamedTemporaryFile(suffix=".rpm") as th:<commit_msg>Add message saying url we are scraping.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License..

import os
import platform
import random
import re
import subprocess
import sys
import tempfile
import time
import urllib2
import contextlib

BOOT_STRAP_FN = os.path.join(os.path.expanduser("~"), ".anvil_strapped")
RH_EPEL_URLS = [
    'http://mirror.cogentco.com/pub/linux/epel/6/i386',
    "http://epel.mirror.freedomvoice.com/6/i386",
    'http://mirrors.kernel.org/fedora-epel/6/i386',
]
URL_TIMEOUT = 5


def tiny_p(cmd, capture=True):
    # Darn python 2.6 doesn't have check_output (argggg)
    stdout = subprocess.PIPE
    stderr = subprocess.PIPE
    if not capture:
        stdout = None
        stderr = None
    sp = subprocess.Popen(cmd, stdout=stdout,
                    stderr=stderr, stdin=None)
    (out, err) = sp.communicate()
    if sp.returncode not in [0]:
        raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" 
                            % (cmd, sp.returncode, out, err))
    return (out, err)


def _write_msg(msg):
    sys.stderr.write("%s\n" % (msg))


def _write_warn(msg):
    _write_msg("WARNING: %s" % (msg))


def _strap_epel():
    exceptions = []
    fetched = False
    _write_msg("Installing EPEL repositories...")
    for u in RH_EPEL_URLS:
        try:
            _write_msg("Scraping %s for the EPEL release rpm filename." % (u))
            with contextlib.closing(urllib2.urlopen(u, timeout=URL_TIMEOUT)) as uh:
                rel = re.search("epel-release(.*?)[.]rpm", uh.read(), re.I)
            if rel:
                with tempfile.NamedTemporaryFile(suffix=".rpm") as th:
{'fraction_non_alphanumeric': 0.11546298392574145, 'fraction_numerical': 0.013131084446456871, 'mean_word_length': 2.8788410886742755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2711641', 'n_tokens_mistral': 1488, 'n_tokens_neox': 1419, 'n_words': 424}