<commit_before>#!/usr/bin/env python
"""Installer for gemini: a lightweight db framework for disease and population genetics.

https://github.com/arq5x/gemini

Handles installation of:

- Required third party software
- Required Python libraries
- Gemini application
- Associated data files

Requires: Python 2.7 (or 2.6 and argparse), git, and compilers (gcc, g++)

Run gemini_install.py -h for usage.
"""
import argparse
import platform
import os
import shutil
import subprocess
import sys
import urllib2
import urllib

remotes = {"requirements_pip": "https://raw.github.com/arq5x/gemini/master/requirements.txt",
           "requirements_conda": "",
           "versioned_installations": "https://raw.githubusercontent.com/arq5x/gemini/master/versioning/",
           "gemini": "https://github.com/arq5x/gemini.git",
           "anaconda": "http://repo.continuum.io/miniconda/Miniconda-3.7.0-%s-x86%s.sh"}

remotes_dev = remotes.copy()
remotes_dev.update({
    "requirements_pip": "https://raw.github.com/arq5x/gemini/dev/requirements.txt",
    "gemini": "git+https://github.com/arq5x/gemini.git@dev",
    "requirements_conda": "https://raw.githubusercontent.com/arq5x/gemini/dev/versioning/unstable/requirements_conda.txt",
})

remotes_bp = remotes_dev
remotes_bp.update({
    "requirements_pip": "https://raw.github.com/brentp/gemini/dev/requirements.txt",
    "gemini": "git+https://github.com/brentp/gemini.git@dev",
    "requirements_conda": "https://raw.githubusercontent.com/brentp/gemini/dev/versioning/unstable/requirements_conda.txt",
})


def main(args, remotes=remotes):
    check_dependencies()
    clean_env_variables()
    work_dir = os.path.join(os.getcwd(), "tmpgemini_install")
    if not os.path.exists(work_dir):
        os.makedirs(work_dir)
    os.chdir(work_dir)

    if args.gemini_version in ("unstable", "bp"):
        if args.gemini_version == "unstable":
            remotes = remotes_dev
        else:
            remotes = remotes_bp

        requirements_pip = remotes['requirements_pip']
        requirements_conda = remotes['requirements_conda']
        urllib.urlretrieve(requirements_pip, filename='_pip_dev.txt')
        urllib.urlretrieve(requirements_conda, filename='_conda_dev.txt')

        # quick hack to support testing installs:
        if args.gemini_version == "bp":
            for f in ('_pip_dev.txt', '_conda_dev.txt'):
                contents = open(f).read().replace('arq5x', 'brentp')
                with open(f, 'w') as fh:
                    fh.write(contents)

        remotes.update({'requirements_pip': '_pip_dev.txt', 'requirements_conda': '_conda_dev.txt'})

    elif args.gemini_version != 'latest':
        requirements_pip = os.path.join(remotes['versioned_installations'],
                                        args.gemini_version, 'requirements_pip.txt')
        requirements_conda = os.path.join(remotes['versioned_installations'],
                                          args.gemini_version, 'requirements_conda.txt')
        try:
            urllib2.urlopen(requirements_pip)
        except:
            sys.exit('Gemini version %s could not be found. Try the latest version.' % args.gemini_version)
        remotes.update({'requirements_pip': requirements_pip, 'requirements_conda': requirements_conda})

    print "Installing isolated base python installation"
    make_dirs(args)
    anaconda = install_anaconda_python(args, remotes)
    print "Installing gemini..."
    install_conda_pkgs(anaconda, remotes, args)
    gemini = install_gemini(anaconda, remotes, args.datadir, args.tooldir, args.sudo)
    if args.install_tools:
        print "Installing associated tools..."
        link_tools(args.tooldir, anaconda["dir"])
    os.chdir(work_dir)
    install_data(gemini["python"], gemini["data_script"], args)
    os.chdir(work_dir)
    test_script = install_testbase(args.datadir, remotes["gemini"], gemini)
    print "Finished: gemini, tools and data installed"
    print " Tools installed in:\n  %s" % args.tooldir
    print " Data installed in:\n  %s" % args.datadir
    print " Run tests with:\n  cd %s && bash %s" % (os.path.dirname(test_script),
                                                    os.path.basename(test_script))
    print " NOTE: be sure to add %s/bin to your PATH." % args.tooldir
    print " NOTE: Install data files for GERP_bp & CADD_scores (not installed by default).\n "

    shutil.rmtree(work_dir)

def link_tools(tooldir, anaconda_dir):
    """Link tools installed via conda into the tool directory.
    """
    bins = ["grabix"]
    for binfile in bins:
        orig_file = os.path.join(anaconda_dir, "bin", binfile)
        final_file = os.path.join(tooldir, "bin", binfile)
        if os.path.exists(orig_file):
            _do_link(orig_file, final_file)

def _do_link(orig_file, final_file):
    """Perform a soft link of the original file into the final location.

    We need the symlink to point to /anaconda/bin directory, not the real location
    in the pkgs directory so conda can resolve LD_LIBRARY_PATH and the interpreters.
    """
    needs_link = True
    # working symlink, check if already in the right place or remove it
    if os.path.exists(final_file):
        if (os.path.realpath(final_file) == os.path.realpath(orig_file) and
              orig_file == os.path.normpath(os.path.join(os.path.dirname(final_file), os.readlink(final_file)))):
            needs_link = False
        else:
            os.remove(final_file)
    # broken symlink
    elif os.path.lexists(final_file):
        os.unlink(final_file)
    if needs_link:
        os.symlink(os.path.relpath(orig_file, os.path.dirname(final_file)), final_file)

def install_gemini(anaconda, remotes, datadir, tooldir, use_sudo):
    """Install gemini plus python dependencies inside isolated Anaconda environment.
    """
    # Work around issue with distribute where asks for 'distribute==0.0'
    # try:
    #     subprocess.check_call([anaconda["easy_install"], "--upgrade", "distribute"])
    # except subprocess.CalledProcessError:
    #     try:
    #         subprocess.check_call([anaconda["pip"], "install", "--upgrade", "distribute"])
    #     except subprocess.CalledProcessError:
    #         pass
    # allow downloads excluded in recent pip (1.5 or greater) versions
    try:
        p = subprocess.Popen([anaconda["pip"], "--version"], stdout=subprocess.PIPE)
        pip_version = p.communicate()[0].split()[1]
    except:
        pip_version = ""
    pip_compat = []
    if pip_version >= "1.5":
        for req in ["python-graph-core", "python-graph-dot"]:
            pip_compat += ["--allow-external", req, "--allow-unverified", req]
    # Set PIP SSL certificate to installed conda certificate to avoid SSL errors
    cert_file = os.path.join(anaconda["dir"], "ssl", "cert.pem")
    if os.path.exists(cert_file):
        os.environ["PIP_CERT"] = cert_file
    subprocess.check_call([anaconda["pip"], "install"] + pip_compat + ["-r", remotes["requirements_pip"]])
    python_bin = os.path.join(anaconda["dir"], "bin", "python")
    _cleanup_problem_files(anaconda["dir"])
    _add_missing_inits(python_bin)
    for final_name, ve_name in [("gemini", "gemini"), ("gemini_python", "python"),
                                ("gemini_pip", "pip")]:
        final_script = os.path.join(tooldir, "bin", final_name)
        ve_script = os.path.join(anaconda["dir"], "bin", ve_name)
        sudo_cmd = ["sudo"] if use_sudo else []
        if os.path.lexists(final_script):
            subprocess.check_call(sudo_cmd + ["rm", "-f", final_script])
        else:
            subprocess.check_call(sudo_cmd + ["mkdir", "-p", os.path.dirname(final_script)])
        cmd = ["ln", "-s", ve_script, final_script]
        subprocess.check_call(sudo_cmd + cmd)
    library_loc = check_output("%s -c 'import gemini; print gemini.__file__'" % python_bin,
                               shell=True)
    return {"fab": os.path.join(anaconda["dir"], "bin", "fab"),
            "data_script": os.path.join(os.path.dirname(library_loc.strip()), "install-data.py"),
            "python": python_bin,
            "cmd": os.path.join(anaconda["dir"], "bin", "gemini")}

def install_conda_pkgs(anaconda, remotes, args):
    if args.gemini_version != 'latest':
        pkgs = ["--file", remotes['requirements_conda']]
    else:
        pkgs = ["bcolz", "conda", "cython", "grabix", "ipyparallel",
                "jinja2", "nose", "numexpr", "numpy", "openssl", "pip", "pybedtools",
                "pycrypto", "pyparsing", "python-graph-core", "python-graph-dot",
                "pyyaml", "pyzmq", "pandas", "scipy"]
        if platform.architecture()[0] != "32bit":
            pkgs += ["bx-python", "pysam", "ipython-cluster-helper"]

    channels = ["-c", "bcbio", "-c", "bioconda"]
    subprocess.check_call([anaconda["conda"], "install", "--yes"] + channels + pkgs)

def install_anaconda_python(args, remotes):
    """Provide isolated installation of Anaconda python.
    http://docs.continuum.io/anaconda/index.html
    """
    anaconda_dir = os.path.join(args.datadir, "anaconda")
    bindir = os.path.join(anaconda_dir, "bin")
    conda = os.path.join(bindir, "conda")
    if platform.mac_ver()[0]:
        distribution = "macosx"
    else:
        distribution = "linux"
    if platform.architecture()[0] == "32bit":
        arch = ""
    else:
        arch = "_64"
    if not os.path.exists(anaconda_dir) or not os.path.exists(conda):
        if os.path.exists(anaconda_dir):
            shutil.rmtree(anaconda_dir)
        url = remotes["anaconda"] % ("MacOSX" if distribution == "macosx" else "Linux", arch)
        if not os.path.exists(os.path.basename(url)):
            subprocess.check_call(["wget", "--continue", url])
        subprocess.check_call("bash %s -b -p %s" %
                              (os.path.basename(url), anaconda_dir), shell=True)
    return {"conda": conda,
            "pip": os.path.join(bindir, "pip"),
            "easy_install": os.path.join(bindir, "easy_install"),
            "dir": anaconda_dir}

def _add_missing_inits(python_bin):
    """pip/setuptools strips __init__.py files with namespace declarations.
    I have no idea why, but this adds them back, skipping if compiled into an egg.
    """
    library_loc = check_output("%s -c 'import pygraph.classes.graph; "
                               "print pygraph.classes.graph.__file__'" % python_bin,
                               shell=True)
    pygraph_init = os.path.normpath(os.path.join(os.path.dirname(library_loc.strip()), os.pardir,
                                                 "__init__.py"))
    if not os.path.exists(pygraph_init):
        if os.path.isdir(os.path.dirname(pygraph_init)):
            with open(pygraph_init, "w") as out_handle:
                out_handle.write("__import__('pkg_resources').declare_namespace(__name__)\n")

def _cleanup_problem_files(venv_dir):
    """Remove problem bottle items in PATH which conflict with site-packages
    """
    for cmd in ["bottle.py", "bottle.pyc"]:
        bin_cmd = os.path.join(venv_dir, "bin", cmd)
        if os.path.exists(bin_cmd):
            os.remove(bin_cmd)

def install_data(python_cmd, data_script, args):
    """Install biological data used by gemini.
    """
    data_dir = os.path.join(args.datadir, "gemini_data") if args.sharedpy else args.datadir
    cmd = [python_cmd, data_script, data_dir]
    if args.install_data:
        print "Installing gemini data..."
    else:
        cmd.append("--nodata")
    subprocess.check_call(cmd)

def install_testbase(datadir, repo, gemini):
    """Clone or update gemini code so we have the latest test suite.
    """
    gemini_dir = os.path.join(datadir, "gemini")
    cur_dir = os.getcwd()
    needs_git = True
    if os.path.exists(gemini_dir):
        os.chdir(gemini_dir)
        try:
            subprocess.check_call(["git", "pull", "origin", "master", "--tags"])
            needs_git = False
        except:
            os.chdir(cur_dir)
            shutil.move(gemini_dir, "gtmp")

    branch = None
    if needs_git:
        os.chdir(os.path.split(gemini_dir)[0])
        if repo.startswith("git+"):
            repo = repo[4:]
        if repo.find("@"):
            url, branch = repo.rsplit("@", 1)
            subprocess.check_call(["git", "clone", "-b", branch, url])
        else:
            subprocess.check_call(["git", "clone", repo])
        os.makedirs(os.path.join(gemini_dir, "data"))
        if os.path.exists(os.path.join(cur_dir, "gtmp", "data")):
            for f in os.listdir(os.path.join(cur_dir, "gtmp", "data")):
                shutil.move(os.path.join(cur_dir, "gtmp", "data", f), os.path.join(gemini_dir, "data"))
            #shutil.move(os.path.join(cur_dir, "gtmp"), gemini_dir)
            shutil.rmtree(os.path.join(cur_dir, "gtmp", "data"))<commit_msg>fix for 32bf46f80<commit_after>#!/usr/bin/env python
"""Installer for gemini: a lightweight db framework for disease and population genetics.

https://github.com/arq5x/gemini

Handles installation of:

- Required third party software
- Required Python libraries
- Gemini application
- Associated data files

Requires: Python 2.7 (or 2.6 and argparse), git, and compilers (gcc, g++)

Run gemini_install.py -h for usage.
"""
import argparse
import platform
import os
import shutil
import subprocess
import sys
import urllib2
import urllib

remotes = {"requirements_pip": "https://raw.github.com/arq5x/gemini/master/requirements.txt",
           "requirements_conda": "",
           "versioned_installations": "https://raw.githubusercontent.com/arq5x/gemini/master/versioning/",
           "gemini": "https://github.com/arq5x/gemini.git",
           "anaconda": "http://repo.continuum.io/miniconda/Miniconda-3.7.0-%s-x86%s.sh"}

remotes_dev = remotes.copy()
remotes_dev.update({
    "requirements_pip": "https://raw.github.com/arq5x/gemini/dev/requirements.txt",
    "gemini": "git+https://github.com/arq5x/gemini.git@dev",
    "requirements_conda": "https://raw.githubusercontent.com/arq5x/gemini/dev/versioning/unstable/requirements_conda.txt",
})

remotes_bp = remotes_dev
remotes_bp.update({
    "requirements_pip": "https://raw.github.com/brentp/gemini/dev/requirements.txt",
    "gemini": "git+https://github.com/brentp/gemini.git@dev",
    "requirements_conda": "https://raw.githubusercontent.com/brentp/gemini/dev/versioning/unstable/requirements_conda.txt",
})


def main(args, remotes=remotes):
    check_dependencies()
    clean_env_variables()
    work_dir = os.path.join(os.getcwd(), "tmpgemini_install")
    if not os.path.exists(work_dir):
        os.makedirs(work_dir)
    os.chdir(work_dir)

    if args.gemini_version in ("unstable", "bp"):
        if args.gemini_version == "unstable":
            remotes = remotes_dev
        else:
            remotes = remotes_bp

        requirements_pip = remotes['requirements_pip']
        requirements_conda = remotes['requirements_conda']
        urllib.urlretrieve(requirements_pip, filename='_pip_dev.txt')
        urllib.urlretrieve(requirements_conda, filename='_conda_dev.txt')

        # quick hack to support testing installs:
        if args.gemini_version == "bp":
            for f in ('_pip_dev.txt', '_conda_dev.txt'):
                contents = open(f).read().replace('arq5x', 'brentp')
                with open(f, 'w') as fh:
                    fh.write(contents)

        remotes.update({'requirements_pip': '_pip_dev.txt', 'requirements_conda': '_conda_dev.txt'})

    elif args.gemini_version != 'latest':
        requirements_pip = os.path.join(remotes['versioned_installations'],
                                        args.gemini_version, 'requirements_pip.txt')
        requirements_conda = os.path.join(remotes['versioned_installations'],
                                          args.gemini_version, 'requirements_conda.txt')
        try:
            urllib2.urlopen(requirements_pip)
        except:
            sys.exit('Gemini version %s could not be found. Try the latest version.' % args.gemini_version)
        remotes.update({'requirements_pip': requirements_pip, 'requirements_conda': requirements_conda})

    print "Installing isolated base python installation"
    make_dirs(args)
    anaconda = install_anaconda_python(args, remotes)
    print "Installing gemini..."
    install_conda_pkgs(anaconda, remotes, args)
    gemini = install_gemini(anaconda, remotes, args.datadir, args.tooldir, args.sudo)
    if args.install_tools:
        print "Installing associated tools..."
        link_tools(args.tooldir, anaconda["dir"])
    os.chdir(work_dir)
    install_data(gemini["python"], gemini["data_script"], args)
    os.chdir(work_dir)
    test_script = install_testbase(args.datadir, remotes["gemini"], gemini)
    print "Finished: gemini, tools and data installed"
    print " Tools installed in:\n  %s" % args.tooldir
    print " Data installed in:\n  %s" % args.datadir
    print " Run tests with:\n  cd %s && bash %s" % (os.path.dirname(test_script),
                                                    os.path.basename(test_script))
    print " NOTE: be sure to add %s/bin to your PATH." % args.tooldir
    print " NOTE: Install data files for GERP_bp & CADD_scores (not installed by default).\n "

    shutil.rmtree(work_dir)

def link_tools(tooldir, anaconda_dir):
    """Link tools installed via conda into the tool directory.
    """
    bins = ["grabix"]
    for binfile in bins:
        orig_file = os.path.join(anaconda_dir, "bin", binfile)
        final_file = os.path.join(tooldir, "bin", binfile)
        if os.path.exists(orig_file):
            _do_link(orig_file, final_file)

def _do_link(orig_file, final_file):
    """Perform a soft link of the original file into the final location.

    We need the symlink to point to /anaconda/bin directory, not the real location
    in the pkgs directory so conda can resolve LD_LIBRARY_PATH and the interpreters.
    """
    needs_link = True
    # working symlink, check if already in the right place or remove it
    if os.path.exists(final_file):
        if (os.path.realpath(final_file) == os.path.realpath(orig_file) and
              orig_file == os.path.normpath(os.path.join(os.path.dirname(final_file), os.readlink(final_file)))):
            needs_link = False
        else:
            os.remove(final_file)
    # broken symlink
    elif os.path.lexists(final_file):
        os.unlink(final_file)
    if needs_link:
        os.symlink(os.path.relpath(orig_file, os.path.dirname(final_file)), final_file)

def install_gemini(anaconda, remotes, datadir, tooldir, use_sudo):
    """Install gemini plus python dependencies inside isolated Anaconda environment.
    """
    # Work around issue with distribute where asks for 'distribute==0.0'
    # try:
    #     subprocess.check_call([anaconda["easy_install"], "--upgrade", "distribute"])
    # except subprocess.CalledProcessError:
    #     try:
    #         subprocess.check_call([anaconda["pip"], "install", "--upgrade", "distribute"])
    #     except subprocess.CalledProcessError:
    #         pass
    # allow downloads excluded in recent pip (1.5 or greater) versions
    try:
        p = subprocess.Popen([anaconda["pip"], "--version"], stdout=subprocess.PIPE)
        pip_version = p.communicate()[0].split()[1]
    except:
        pip_version = ""
    pip_compat = []
    if pip_version >= "1.5":
        for req in ["python-graph-core", "python-graph-dot"]:
            pip_compat += ["--allow-external", req, "--allow-unverified", req]
    # Set PIP SSL certificate to installed conda certificate to avoid SSL errors
    cert_file = os.path.join(anaconda["dir"], "ssl", "cert.pem")
    if os.path.exists(cert_file):
        os.environ["PIP_CERT"] = cert_file
    subprocess.check_call([anaconda["pip"], "install"] + pip_compat + ["-r", remotes["requirements_pip"]])
    python_bin = os.path.join(anaconda["dir"], "bin", "python")
    _cleanup_problem_files(anaconda["dir"])
    _add_missing_inits(python_bin)
    for final_name, ve_name in [("gemini", "gemini"), ("gemini_python", "python"),
                                ("gemini_pip", "pip")]:
        final_script = os.path.join(tooldir, "bin", final_name)
        ve_script = os.path.join(anaconda["dir"], "bin", ve_name)
        sudo_cmd = ["sudo"] if use_sudo else []
        if os.path.lexists(final_script):
            subprocess.check_call(sudo_cmd + ["rm", "-f", final_script])
        else:
            subprocess.check_call(sudo_cmd + ["mkdir", "-p", os.path.dirname(final_script)])
        cmd = ["ln", "-s", ve_script, final_script]
        subprocess.check_call(sudo_cmd + cmd)
    library_loc = check_output("%s -c 'import gemini; print gemini.__file__'" % python_bin,
                               shell=True)
    return {"fab": os.path.join(anaconda["dir"], "bin", "fab"),
            "data_script": os.path.join(os.path.dirname(library_loc.strip()), "install-data.py"),
            "python": python_bin,
            "cmd": os.path.join(anaconda["dir"], "bin", "gemini")}

def install_conda_pkgs(anaconda, remotes, args):
    if args.gemini_version != 'latest':
        pkgs = ["--file", remotes['requirements_conda']]
    else:
        pkgs = ["bcolz", "conda", "cython", "grabix", "ipyparallel",
                "jinja2", "nose", "numexpr", "numpy", "openssl", "pip", "pybedtools",
                "pycrypto", "pyparsing", "python-graph-core", "python-graph-dot",
                "pyyaml", "pyzmq", "pandas", "scipy"]
        if platform.architecture()[0] != "32bit":
            pkgs += ["bx-python", "pysam", "ipython-cluster-helper"]

    channels = ["-c", "bcbio", "-c", "bioconda"]
    subprocess.check_call([anaconda["conda"], "install", "--yes"] + channels + pkgs)

def install_anaconda_python(args, remotes):
    """Provide isolated installation of Anaconda python.
    http://docs.continuum.io/anaconda/index.html
    """
    anaconda_dir = os.path.join(args.datadir, "anaconda")
    bindir = os.path.join(anaconda_dir, "bin")
    conda = os.path.join(bindir, "conda")
    if platform.mac_ver()[0]:
        distribution = "macosx"
    else:
        distribution = "linux"
    if platform.architecture()[0] == "32bit":
        arch = ""
    else:
        arch = "_64"
    if not os.path.exists(anaconda_dir) or not os.path.exists(conda):
        if os.path.exists(anaconda_dir):
            shutil.rmtree(anaconda_dir)
        url = remotes["anaconda"] % ("MacOSX" if distribution == "macosx" else "Linux", arch)
        if not os.path.exists(os.path.basename(url)):
            subprocess.check_call(["wget", "--continue", url])
        subprocess.check_call("bash %s -b -p %s" %
                              (os.path.basename(url), anaconda_dir), shell=True)
    return {"conda": conda,
            "pip": os.path.join(bindir, "pip"),
            "easy_install": os.path.join(bindir, "easy_install"),
            "dir": anaconda_dir}

def _add_missing_inits(python_bin):
    """pip/setuptools strips __init__.py files with namespace declarations.
    I have no idea why, but this adds them back, skipping if compiled into an egg.
    """
    library_loc = check_output("%s -c 'import pygraph.classes.graph; "
                               "print pygraph.classes.graph.__file__'" % python_bin,
                               shell=True)
    pygraph_init = os.path.normpath(os.path.join(os.path.dirname(library_loc.strip()), os.pardir,
                                                 "__init__.py"))
    if not os.path.exists(pygraph_init):
        if os.path.isdir(os.path.dirname(pygraph_init)):
            with open(pygraph_init, "w") as out_handle:
                out_handle.write("__import__('pkg_resources').declare_namespace(__name__)\n")

def _cleanup_problem_files(venv_dir):
    """Remove problem bottle items in PATH which conflict with site-packages
    """
    for cmd in ["bottle.py", "bottle.pyc"]:
        bin_cmd = os.path.join(venv_dir, "bin", cmd)
        if os.path.exists(bin_cmd):
            os.remove(bin_cmd)

def install_data(python_cmd, data_script, args):
    """Install biological data used by gemini.
    """
    data_dir = os.path.join(args.datadir, "gemini_data") if args.sharedpy else args.datadir
    cmd = [python_cmd, data_script, data_dir]
    if args.install_data:
        print "Installing gemini data..."
    else:
        cmd.append("--nodata")
    subprocess.check_call(cmd)

def install_testbase(datadir, repo, gemini):
    """Clone or update gemini code so we have the latest test suite.
    """
    gemini_dir = os.path.join(datadir, "gemini")
    cur_dir = os.getcwd()
    needs_git = True
    if os.path.exists(gemini_dir):
        os.chdir(gemini_dir)
        try:
            subprocess.check_call(["git", "pull", "origin", "master", "--tags"])
            needs_git = False
        except:
            os.chdir(cur_dir)
            shutil.move(gemini_dir, "gtmp")

    branch = None
    if needs_git:
        os.chdir(os.path.split(gemini_dir)[0])
        if repo.startswith("git+"):
            repo = repo[4:]
        if "@" in repo:
            url, branch = repo.rsplit("@", 1)
            subprocess.check_call(["git", "clone", "-b", branch, url])
        else:
            subprocess.check_call(["git", "clone", repo])
        os.makedirs(os.path.join(gemini_dir, "data"))
        if os.path.exists(os.path.join(cur_dir, "gtmp", "data")):
            for f in os.listdir(os.path.join(cur_dir, "gtmp", "data")):
                shutil.move(os.path.join(cur_dir, "gtmp", "data", f), os.path.join(gemini_dir, "data"))
            #shutil.move(os.path.join(cur_dir, "gtmp"), gemini_dir)
            shutil.rmtree(os.path.join(cur_dir, "gtmp", "data"))
{'fraction_non_alphanumeric': 0.14041723532852407, 'fraction_numerical': 0.003353480210567362, 'mean_word_length': 2.8635131063573365, 'pattern_counts': {'":': 48, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 20, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6416373', 'n_tokens_mistral': 8578, 'n_tokens_neox': 8385, 'n_words': 1724}