<commit_before>#!/usr/bin/env python2.7

import os
import sys
import socket
from fabric.tasks import Task
from fabric.api import env, task, settings, hide, execute, local, sudo, abort, warn, put, run, lcd
from fabric.colors import red, cyan, green, yellow
from fabric.contrib.console import confirm


# don't import anything into public when someone loads us
__all__ = []


# colorize errors because!
env.colorize_errors = True


# define paths for doing work
env.current_dir     = os.getcwd()
env.push_dir        = os.path.dirname(os.path.realpath(sys.argv[0]))
env.containment_dir = "{}/.push".format(env.current_dir)
env.build_dir       = "{}/build".format(env.containment_dir)
env.test_dir        = "{}/test".format(env.containment_dir)
env.archive_dir     = "{}/archive".format(env.containment_dir)
env.release_dir     = "{}/release".format(env.containment_dir)
env.temp_dir        = "{}/temp".format(env.containment_dir)


# keep track of what tasks we have run so that we don't run them again
env.completed_tasks = {}


# the path to the root of the git repository
with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True):
    env.git_root_dir = local("{} rev-parse --show-toplevel".format(env.git), capture=True)


# get the latest commit/tag and branch of the repo or HEAD if no commit/tag and/or branch
with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True):
    if (int(local("ls {}/.git/refs/heads/ | wc -l | tr -d ' '".format(env.git_root_dir), capture=True)) != 0):
        env.repo_commit_name = local("{} describe --always --tags".format(env.git), capture=True).strip()
        env.repo_branch_name = local("{} rev-parse --abbrev-ref HEAD".format(env.git), capture=True).strip()
        env.repo_tag_name = local("{} describe --tags --exact-match".format(env.git), capture=True).strip()

    if ("repo_commit_name" not in env or env.repo_commit_name is None or env.repo_commit_name == ""):
        env.repo_commit_name = "HEAD"
    if ("repo_branch_name" not in env or env.repo_branch_name is None or env.repo_commit_name == ""):
        env.repo_branch_name = "HEAD"
    if ("repo_tag_name" not in env or env.repo_tag_name is None or env.repo_tag_name == ""):
        env.repo_tag_name = None


# is set to "true" if the repository is dirty
with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True):
    if (str(local("{} status -s".format(env.git), capture=True)) != ""):
        env.repo_is_dirty = True
    else:
        env.repo_is_dirty = False


# the name of the current directory, used as the tool name when doing certain tasks
env.project_name = os.path.basename(os.path.normpath(os.getcwd()))


# the name of the archive we will create when asked to create the archive
# if the thing is dirty the append "dirty" to it
env.repo_version = "{}{}".format(env.repo_commit_name, ("-dirty" if env.repo_is_dirty else ""))
env.archive_name = "{}-version-{}.tar.gz".format(env.project_name, env.repo_version)


# flags for tar and rsync
env.tar_c_flags = "-p"
env.tar_x_flags = "-p --no-same-owner --overwrite-dir"
env.rsync_flags = "-aH --numeric-ids --exclude=.git --exclude=.gitignore --exclude={git_root}/.gitignore --exclude-from=.gitignore --exclude-from={git_root}/.gitignore".format(git_root=env.git_root_dir)


class CleanTask(Task):
    """
        removes all built content
    """

    name = "clean"

    def run(self):
        # don't run this class more than once
        if (self.__class__.__name__ in env.completed_tasks and env.completed_tasks[self.__class__.__name__]):
            return

        local("rm -rf {}".format(env.containment_dir))
        print(green("Finished cleaning project."))

        # record that we've run this step
        env.completed_tasks[self.__class__.__name__] = True


class MostlyCleanTask(Task):
    """
        remove all build artifacts except test output
    """

    name = "mostlyclean"

    def run(self):
        # don't run this class more than once
        if (self.__class__.__name__ in env.completed_tasks and env.completed_tasks[self.__class__.__name__]):
            return

        local("rm -rf {}".format(env.build_dir))
        local("rm -rf {}".format(env.release_dir))
        local("rm -rf {}".format(env.archive_dir))
        local("rm -rf {}".format(env.temp_dir))
        print(green("Finished mostly cleaning project."))
<commit_msg>clean should allow running tasks again<commit_after>#!/usr/bin/env python2.7

import os
import sys
import socket
from fabric.tasks import Task
from fabric.api import env, task, settings, hide, execute, local, sudo, abort, warn, put, run, lcd
from fabric.colors import red, cyan, green, yellow
from fabric.contrib.console import confirm


# don't import anything into public when someone loads us
__all__ = []


# colorize errors because!
env.colorize_errors = True


# define paths for doing work
env.current_dir     = os.getcwd()
env.push_dir        = os.path.dirname(os.path.realpath(sys.argv[0]))
env.containment_dir = "{}/.push".format(env.current_dir)
env.build_dir       = "{}/build".format(env.containment_dir)
env.test_dir        = "{}/test".format(env.containment_dir)
env.archive_dir     = "{}/archive".format(env.containment_dir)
env.release_dir     = "{}/release".format(env.containment_dir)
env.temp_dir        = "{}/temp".format(env.containment_dir)


# keep track of what tasks we have run so that we don't run them again
env.completed_tasks = {}


# the path to the root of the git repository
with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True):
    env.git_root_dir = local("{} rev-parse --show-toplevel".format(env.git), capture=True)


# get the latest commit/tag and branch of the repo or HEAD if no commit/tag and/or branch
with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True):
    if (int(local("ls {}/.git/refs/heads/ | wc -l | tr -d ' '".format(env.git_root_dir), capture=True)) != 0):
        env.repo_commit_name = local("{} describe --always --tags".format(env.git), capture=True).strip()
        env.repo_branch_name = local("{} rev-parse --abbrev-ref HEAD".format(env.git), capture=True).strip()
        env.repo_tag_name = local("{} describe --tags --exact-match".format(env.git), capture=True).strip()

    if ("repo_commit_name" not in env or env.repo_commit_name is None or env.repo_commit_name == ""):
        env.repo_commit_name = "HEAD"
    if ("repo_branch_name" not in env or env.repo_branch_name is None or env.repo_commit_name == ""):
        env.repo_branch_name = "HEAD"
    if ("repo_tag_name" not in env or env.repo_tag_name is None or env.repo_tag_name == ""):
        env.repo_tag_name = None


# is set to "true" if the repository is dirty
with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True):
    if (str(local("{} status -s".format(env.git), capture=True)) != ""):
        env.repo_is_dirty = True
    else:
        env.repo_is_dirty = False


# the name of the current directory, used as the tool name when doing certain tasks
env.project_name = os.path.basename(os.path.normpath(os.getcwd()))


# the name of the archive we will create when asked to create the archive
# if the thing is dirty the append "dirty" to it
env.repo_version = "{}{}".format(env.repo_commit_name, ("-dirty" if env.repo_is_dirty else ""))
env.archive_name = "{}-version-{}.tar.gz".format(env.project_name, env.repo_version)


# flags for tar and rsync
env.tar_c_flags = "-p"
env.tar_x_flags = "-p --no-same-owner --overwrite-dir"
env.rsync_flags = "-aH --numeric-ids --exclude=.git --exclude=.gitignore --exclude={git_root}/.gitignore --exclude-from=.gitignore --exclude-from={git_root}/.gitignore".format(git_root=env.git_root_dir)


class CleanTask(Task):
    """
        removes all built content
    """

    name = "clean"

    def run(self):
        # clean absolutely everything
        env.completed_tasks = {}

        local("rm -rf {}".format(env.containment_dir))
        print(green("Finished cleaning project."))

        # record that we've run this step
        env.completed_tasks[self.__class__.__name__] = True


class MostlyCleanTask(Task):
    """
        remove all build artifacts except test output
    """

    name = "mostlyclean"

    def run(self):
        # clean absolutely everything
        env.completed_tasks = {}

        local("rm -rf {}".format(env.build_dir))
        local("rm -rf {}".format(env.release_dir))
        local("rm -rf {}".format(env.archive_dir))
        local("rm -rf {}".format(env.temp_dir))
        print(green("Finished mostly cleaning project."))

{'fraction_non_alphanumeric': 0.14294015107495642, 'fraction_numerical': 0.0009296920395119117, 'mean_word_length': 3.5582627118644066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1036847', 'n_tokens_mistral': 2820, 'n_tokens_neox': 2719, 'n_words': 733}