<commit_before>from fabric.api import *
from vars import *
from urlparse import urlparse
from vars import *
import os
import re

@task
def is_installed(package):
    """Check if package is installed"""
    vars = Vars()
    # This will work with "package.<extension> and "package"
    package_name = os.path.splitext(package)[0]
    with quiet(), settings(warn_only=True):
        if sudo(vars.os.package_installed_cmd + " " + package_name).failed:
            return False
        else:
            return True

@task
def download_and_install(url):
    """Download a package and install it"""
    vars = Vars()
    with cd("/tmp"):
        parsed = urlparse(url)
        package_file = re.split("/", parsed.path)[1]
        package_name = package_file.rpartition(".")[0]

        if not exists(package_file):
            run("wget "+url)
        if not is_installed(package_name):
            sudo(vars.os.package_local_install_cmd % package_file)

@task
def install(package):
    """Install a package from the repositories"""
    vars = Vars()
    if not is_installed(package):
        sudo(vars.os.package_refresh_cmd)
        sudo(vars.os.package_install_cmd % package)

@task
def autoremove():
    """Remove obsolete packages, such as unused kernel images"""
    vars = Vars()
    with settings(hide("user", "stdout")):
        sudo(vars.os.package_autoremove_cmd)<commit_msg>Allow overriding (installed) package_name in package.download_and_install<commit_after>from fabric.api import *
from vars import *
from urlparse import urlparse
from vars import *
import os
import re

@task
def is_installed(package):
    """Check if package is installed"""
    vars = Vars()
    # This will work with "package.<extension> and "package"
    package_name = os.path.splitext(package)[0]
    with quiet(), settings(warn_only=True):
        if sudo(vars.os.package_installed_cmd + " " + package_name).failed:
            return False
        else:
            return True

@task
def download_and_install(url, package_name=None):
    """Download a package from URL and install it. Use package_name to manually define the name of the installed package and to prevent unnecessary reinstalls."""
    vars = Vars()
    with cd("/tmp"):
        parsed = urlparse(url)
        package_file = re.split("/", parsed.path)[1]

        if not package_name:
            package_name = package_file.rpartition(".")[0]

        if not exists(package_file):
            run("wget "+url)
        if not is_installed(package_name):
            sudo(vars.os.package_local_install_cmd % package_file)

@task
def install(package):
    """Install a package from the repositories"""
    vars = Vars()
    if not is_installed(package):
        sudo(vars.os.package_refresh_cmd)
        sudo(vars.os.package_install_cmd % package)

@task
def autoremove():
    """Remove obsolete packages, such as unused kernel images"""
    vars = Vars()
    with settings(hide("user", "stdout")):
        sudo(vars.os.package_autoremove_cmd)
{'fraction_non_alphanumeric': 0.11155644622578491, 'fraction_numerical': 0.002004008016032064, 'mean_word_length': 3.1597222222222223, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1483536', 'n_tokens_mistral': 895, 'n_tokens_neox': 866, 'n_words': 248}