<commit_before>import setuptools
from setuptools.command.install import install
from distutils.extension import Extension
import os
import shutil
import subprocess
import sys
import tarfile

import urllib

SMAC_DOWNLOAD_LOCATION = "http://aad.informatik.uni-freiburg.de/~feurerm/"
SMAC_TAR_NAME = "smac-v2.08.01-development-1.tar.gz"
METADATA_LOCATION = "http://aad.informatik.uni-freiburg.de/~feurerm/"
METADATA_TAR_NAME = "metadata_automl1_000.tar.gz"
RUNSOLVER_LOCATION = "http://www.cril.univ-artois.fr/~roussel/runsolver/"
RUNSOLVER_TAR_NAME = "runsolver-3.3.4.tar.bz2"
DOWNLOAD_DIRECTORY = os.path.join(os.path.dirname(__file__), ".downloads")
BINARIES_DIRECTORY = "autosklearn/binaries"
METADATA_DIRECTORY = "autosklearn/metalearning/files"

class Download(install):
    def run(self):
        try:
            shutil.rmtree(DOWNLOAD_DIRECTORY)
        except:
            pass

        try:
            os.makedirs(DOWNLOAD_DIRECTORY)
        except:
            pass

        for download_url, filename in [(SMAC_DOWNLOAD_LOCATION, SMAC_TAR_NAME),
                                       (METADATA_LOCATION, METADATA_TAR_NAME),
                                       (RUNSOLVER_LOCATION, RUNSOLVER_TAR_NAME)]:
            # This can fail ungracefully, because having these files is
            # crucial to AutoSklearn!
            urllib.urlretrieve(os.path.join(download_url, filename),
                               filename=os.path.join(DOWNLOAD_DIRECTORY, filename))

            tfile = tarfile.open(os.path.join(DOWNLOAD_DIRECTORY, filename))
            tfile.extractall(os.path.join(DOWNLOAD_DIRECTORY,
                                          filename.replace(".tar.gz", "").
                                          replace(".tar.bz2", "")))

        # Build the runsolver
        sys.stdout.write("Building runsolver\n")
        cur_pwd = os.getcwd()
        runsolver_source_path = os.path.join(DOWNLOAD_DIRECTORY,
                                             "runsolver-3.3.4",
                                             "runsolver", "src")
        os.chdir(runsolver_source_path)
        subprocess.check_call("make")
        os.chdir(cur_pwd)

        # Create a fresh binaries directory
        try:
            shutil.rmtree(BINARIES_DIRECTORY)
        except:
            pass

        try:
            os.makedirs(BINARIES_DIRECTORY)
            with open(os.path.join(BINARIES_DIRECTORY, '__init__.py')):
                pass
        except:
            pass

        # Copy the runsolver into the sources so it gets copied
        shutil.move(os.path.join(runsolver_source_path, "runsolver"),
                    os.path.join(BINARIES_DIRECTORY, "runsolver"))

        # Copy SMAC
        shutil.move(os.path.join(DOWNLOAD_DIRECTORY,
                                 SMAC_TAR_NAME.replace(".tar.gz", "")),
                    BINARIES_DIRECTORY)

        try:
            shutil.rmtree(METADATA_DIRECTORY)
        except:
            pass

        # Copy the metadata
        shutil.move(os.path.join(DOWNLOAD_DIRECTORY,
                                 METADATA_TAR_NAME.replace(".tar.gz", ""),
                                 "files"),
                    METADATA_DIRECTORY)

        # TODO: Normally one wants to call run(self), but this runs distutils and ignores install_requirements for unknown reasons
        # if anyone knows a better way, feel free to change
        install.do_egg_install(self)

        shutil.rmtree(os.path.join(METADATA_DIRECTORY))
        shutil.rmtree(BINARIES_DIRECTORY)
        shutil.rmtree(DOWNLOAD_DIRECTORY)
<commit_msg>fix bug removing all metadata files<commit_after>import setuptools
from setuptools.command.install import install
from distutils.extension import Extension
import os
import shutil
import subprocess
import sys
import tarfile

import urllib

SMAC_DOWNLOAD_LOCATION = "http://aad.informatik.uni-freiburg.de/~feurerm/"
SMAC_TAR_NAME = "smac-v2.08.01-development-1.tar.gz"
#METADATA_LOCATION = "http://aad.informatik.uni-freiburg.de/~feurerm/"
#METADATA_TAR_NAME = "metadata_automl1_000.tar.gz"
RUNSOLVER_LOCATION = "http://www.cril.univ-artois.fr/~roussel/runsolver/"
RUNSOLVER_TAR_NAME = "runsolver-3.3.4.tar.bz2"
DOWNLOAD_DIRECTORY = os.path.join(os.path.dirname(__file__), ".downloads")
BINARIES_DIRECTORY = "autosklearn/binaries"
METADATA_DIRECTORY = "autosklearn/metalearning/files"

class Download(install):
    def run(self):
        try:
            shutil.rmtree(DOWNLOAD_DIRECTORY)
        except:
            pass

        try:
            os.makedirs(DOWNLOAD_DIRECTORY)
        except:
            pass

        for download_url, filename in [(SMAC_DOWNLOAD_LOCATION, SMAC_TAR_NAME),
                                       #(METADATA_LOCATION, METADATA_TAR_NAME),
                                       (RUNSOLVER_LOCATION, RUNSOLVER_TAR_NAME)]:
            # This can fail ungracefully, because having these files is
            # crucial to AutoSklearn!
            urllib.urlretrieve(os.path.join(download_url, filename),
                               filename=os.path.join(DOWNLOAD_DIRECTORY, filename))

            tfile = tarfile.open(os.path.join(DOWNLOAD_DIRECTORY, filename))
            tfile.extractall(os.path.join(DOWNLOAD_DIRECTORY,
                                          filename.replace(".tar.gz", "").
                                          replace(".tar.bz2", "")))

        # Build the runsolver
        sys.stdout.write("Building runsolver\n")
        cur_pwd = os.getcwd()
        runsolver_source_path = os.path.join(DOWNLOAD_DIRECTORY,
                                             "runsolver-3.3.4",
                                             "runsolver", "src")
        os.chdir(runsolver_source_path)
        subprocess.check_call("make")
        os.chdir(cur_pwd)

        # Create a fresh binaries directory
        try:
            shutil.rmtree(BINARIES_DIRECTORY)
        except:
            pass

        try:
            os.makedirs(BINARIES_DIRECTORY)
            with open(os.path.join(BINARIES_DIRECTORY, '__init__.py')):
                pass
        except:
            pass

        # Copy the runsolver into the sources so it gets copied
        shutil.move(os.path.join(runsolver_source_path, "runsolver"),
                    os.path.join(BINARIES_DIRECTORY, "runsolver"))

        # Copy SMAC
        shutil.move(os.path.join(DOWNLOAD_DIRECTORY,
                                 SMAC_TAR_NAME.replace(".tar.gz", "")),
                    BINARIES_DIRECTORY)

        #try:
        #    shutil.rmtree(METADATA_DIRECTORY)
        #except:
        #    pass

        # Copy the metadata
        #shutil.move(os.path.join(DOWNLOAD_DIRECTORY,
        #                         METADATA_TAR_NAME.replace(".tar.gz", ""),
        #                         "files"),
        #            METADATA_DIRECTORY)

        # TODO: Normally one wants to call run(self), but this runs distutils and ignores install_requirements for unknown reasons
        # if anyone knows a better way, feel free to change
        install.do_egg_install(self)

        #shutil.rmtree(os.path.join(METADATA_DIRECTORY))
        shutil.rmtree(BINARIES_DIRECTORY)
        shutil.rmtree(DOWNLOAD_DIRECTORY)

{'fraction_non_alphanumeric': 0.09299750899529477, 'fraction_numerical': 0.00498200941046222, 'mean_word_length': 1.9306569343065694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '579185', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2138, 'n_words': 361}