<commit_before>"""
This is a tool for managing a TKP project. It can be used to initialize a TKP
project and the jobs inside a project. To start using this tool you first
create a TRAP project by running:

  $ tkp-manage.py initproject <projectname>

In the folder where you want to put the TRAP project. To learn more about a
specific `tkp-manage.py` subcommand, run:

  $ tkp-manage.py <subcommand> -h

Have a nice day!
"""
import argparse
import os
from os import path
import shutil
import re
import errno
import getpass
import stat
import sys
import logging

import tkp
from tkp.db.sql.populate import populate


logging.basicConfig(level=logging.INFO)

class CommandError(Exception):
    """
    Exception class indicating a problem while executing a management
    command.

    If this exception is raised during the execution of a management
    command, it will be caught and turned into a nicely-printed error
    message to the appropriate output stream (i.e., stderr); as a
    result, raising this exception (with a sensible description of the
    error) is the preferred way to indicate that something has gone
    wrong in the execution of a command.

    """
    pass


def check_if_exists(filename):
    if not os.access(filename, os.R_OK):
        raise CommandError("can't read %s" % filename)
    return True


def get_template_dir():
    """
    Determines where the job and project templates are.
    """
    return path.join(tkp.__path__[0], 'conf')


def make_writeable(filename):
    """
    Make sure that the file is writeable.
    Useful if our source is read-only.
    """
    if not os.access(filename, os.W_OK):
        st = os.stat(filename)
        new_permissions = stat.S_IMODE(st.st_mode) | stat.S_IWUSR
        os.chmod(filename, new_permissions)


def line_replace(substitutes, line):
    """substitutions - list of 2 item tuples.
    every occurance of the first item in the tuple in line will be replaces
    with the second. The to be replaced thingy should be in Django
    template format e.g. {% user_name %}
    template format e.g. {% user_name %}
    """
    for pattern, repl in substitutes:
        line = re.sub("{%\s*" + pattern + "\s*%}", repl, line)
    return line


def copy_template(job_or_project, name, target=None, **options):
    """
    this is taken from django/core/management/templates.py and modified to fit
    our needs
    """
    # If it's not a valid directory name.
    if not re.search(r'^[_a-zA-Z][\w\-]*$', name):
        # Provide a smart error message, depending on the error.
        if not re.search(r'^[_a-zA-Z]', name):
            message = ('make sure the name begins '
                       'with a letter or underscore')
        else:
            message = 'use only numbers, letters and underscores'
        raise CommandError("%r is not a valid %s name. Please %s." %
                           (name, job_or_project, message))

    # if some directory is given, make sure it's nicely expanded
    if target is None:
        top_dir = path.join(os.getcwd(), name)
        try:
            os.makedirs(top_dir)
        except OSError, e:
            if e.errno == errno.EEXIST:
                message = "'%s' already exists" % top_dir
            else:
                message = e
            raise CommandError(message)
    else:
        top_dir = os.path.abspath(path.expanduser(target))
        if not os.path.exists(top_dir):
            raise CommandError("Destination directory '%s' does not "
                               "exist, please create it first." % top_dir)

    base_name = '%s_name' % job_or_project
    base_subdir = '%s_template' % job_or_project
    base_directory = '%s_directory' % job_or_project

    template_dir = os.path.join(get_template_dir(), base_subdir)
    prefix_length = len(template_dir) + 1

    # these are used for string replacement in templates
    substitutes = (
        ("user_name", getpass.getuser()),
        #("runtime_directory", top_dir)
        )

    for root, dirs, files in os.walk(template_dir):

        path_rest = root[prefix_length:]
        relative_dir = path_rest.replace(base_name, name)
        if relative_dir:
            target_dir = path.join(top_dir, relative_dir)
            if not path.exists(target_dir):
                os.mkdir(target_dir)

        for dirname in dirs[:]:
            if dirname.startswith('.'):
                dirs.remove(dirname)

        for filename in files:
            if filename.endswith(('.pyo', '.pyc', '.py.class')):
                # Ignore some files as they cause various breakages.
                continue
            old_path = path.join(root, filename)
            new_path = path.join(top_dir, relative_dir,
                filename.replace(base_name, name))
            if path.exists(new_path):
                raise CommandError("%s already exists, overlaying a "
                                   "project or job into an existing "
                                   "directory won't replace conflicting "
                                   "files" % new_path)

            with open(old_path, 'r') as template_file:
                content = template_file.read()

            if content:
                content = line_replace(substitutes, content)

            with open(new_path, 'w') as new_file:
                new_file.write(content)

            print "Creating %s" % new_path
            try:
                shutil.copymode(old_path, new_path)
                make_writeable(new_path)
            except OSError:
                sys.stderr.write("Notice: Couldn't set permission bits on %s. You're "
                                  "probably using an uncommon filesystem setup. No "
                                  "problem.\n" % new_path)
    return top_dir


def init_project(args):
    print "creating project '%s'" % args.name
    return copy_template("project", args.name, args.target)


def init_job(args):
    print "creating job '%s'" % args.name
    return copy_template("job", args.name)


def prepare_job(jobname, debug=False):
    here = os.getcwd()
    jobdir = os.path.join(here, jobname)
    pipelinefile = os.path.join(here, "pipeline.cfg")
    sys.path.append(jobdir)
    if debug:
        # show us DEBUG logging
        sys.argv.append("-d")
    else:
        # show us INFO logging
        sys.argv.append("-v")
    # the lofar pipeline utils parse sys.argv to determine some options
    sys.argv += ["-c", pipelinefile, "-j", jobname]


def celery_cmd(args):
    from celery.bin import celery
    base = celery.CeleryCommand(app='tkp.distribute.celery.tasks.trap')
    base.execute_from_commandline(sys.argv[1:])


def run_job(args):
    print "running job '%s'" % args.name<commit_msg>Remove dead code from management.py referring to LofarPipe.<commit_after>"""
This is a tool for managing a TKP project. It can be used to initialize a TKP
project and the jobs inside a project. To start using this tool you first
create a TRAP project by running:

  $ tkp-manage.py initproject <projectname>

In the folder where you want to put the TRAP project. To learn more about a
specific `tkp-manage.py` subcommand, run:

  $ tkp-manage.py <subcommand> -h

Have a nice day!
"""
import argparse
import os
from os import path
import shutil
import re
import errno
import getpass
import stat
import sys
import logging

import tkp
from tkp.db.sql.populate import populate


logging.basicConfig(level=logging.INFO)

class CommandError(Exception):
    """
    Exception class indicating a problem while executing a management
    command.

    If this exception is raised during the execution of a management
    command, it will be caught and turned into a nicely-printed error
    message to the appropriate output stream (i.e., stderr); as a
    result, raising this exception (with a sensible description of the
    error) is the preferred way to indicate that something has gone
    wrong in the execution of a command.

    """
    pass


def check_if_exists(filename):
    if not os.access(filename, os.R_OK):
        raise CommandError("can't read %s" % filename)
    return True


def get_template_dir():
    """
    Determines where the job and project templates are.
    """
    return path.join(tkp.__path__[0], 'conf')


def make_writeable(filename):
    """
    Make sure that the file is writeable.
    Useful if our source is read-only.
    """
    if not os.access(filename, os.W_OK):
        st = os.stat(filename)
        new_permissions = stat.S_IMODE(st.st_mode) | stat.S_IWUSR
        os.chmod(filename, new_permissions)


def line_replace(substitutes, line):
    """substitutions - list of 2 item tuples.
    every occurance of the first item in the tuple in line will be replaces
    with the second. The to be replaced thingy should be in Django
    template format e.g. {% user_name %}
    template format e.g. {% user_name %}
    """
    for pattern, repl in substitutes:
        line = re.sub("{%\s*" + pattern + "\s*%}", repl, line)
    return line


def copy_template(job_or_project, name, target=None, **options):
    """
    this is taken from django/core/management/templates.py and modified to fit
    our needs
    """
    # If it's not a valid directory name.
    if not re.search(r'^[_a-zA-Z][\w\-]*$', name):
        # Provide a smart error message, depending on the error.
        if not re.search(r'^[_a-zA-Z]', name):
            message = ('make sure the name begins '
                       'with a letter or underscore')
        else:
            message = 'use only numbers, letters and underscores'
        raise CommandError("%r is not a valid %s name. Please %s." %
                           (name, job_or_project, message))

    # if some directory is given, make sure it's nicely expanded
    if target is None:
        top_dir = path.join(os.getcwd(), name)
        try:
            os.makedirs(top_dir)
        except OSError, e:
            if e.errno == errno.EEXIST:
                message = "'%s' already exists" % top_dir
            else:
                message = e
            raise CommandError(message)
    else:
        top_dir = os.path.abspath(path.expanduser(target))
        if not os.path.exists(top_dir):
            raise CommandError("Destination directory '%s' does not "
                               "exist, please create it first." % top_dir)

    base_name = '%s_name' % job_or_project
    base_subdir = '%s_template' % job_or_project
    base_directory = '%s_directory' % job_or_project

    template_dir = os.path.join(get_template_dir(), base_subdir)
    prefix_length = len(template_dir) + 1

    # these are used for string replacement in templates
    substitutes = (
        ("user_name", getpass.getuser()),
        #("runtime_directory", top_dir)
        )

    for root, dirs, files in os.walk(template_dir):

        path_rest = root[prefix_length:]
        relative_dir = path_rest.replace(base_name, name)
        if relative_dir:
            target_dir = path.join(top_dir, relative_dir)
            if not path.exists(target_dir):
                os.mkdir(target_dir)

        for dirname in dirs[:]:
            if dirname.startswith('.'):
                dirs.remove(dirname)

        for filename in files:
            if filename.endswith(('.pyo', '.pyc', '.py.class')):
                # Ignore some files as they cause various breakages.
                continue
            old_path = path.join(root, filename)
            new_path = path.join(top_dir, relative_dir,
                filename.replace(base_name, name))
            if path.exists(new_path):
                raise CommandError("%s already exists, overlaying a "
                                   "project or job into an existing "
                                   "directory won't replace conflicting "
                                   "files" % new_path)

            with open(old_path, 'r') as template_file:
                content = template_file.read()

            if content:
                content = line_replace(substitutes, content)

            with open(new_path, 'w') as new_file:
                new_file.write(content)

            print "Creating %s" % new_path
            try:
                shutil.copymode(old_path, new_path)
                make_writeable(new_path)
            except OSError:
                sys.stderr.write("Notice: Couldn't set permission bits on %s. You're "
                                  "probably using an uncommon filesystem setup. No "
                                  "problem.\n" % new_path)
    return top_dir


def init_project(args):
    print "creating project '%s'" % args.name
    return copy_template("project", args.name, args.target)


def init_job(args):
    print "creating job '%s'" % args.name
    return copy_template("job", args.name)


def prepare_job(jobname, debug=False):
    here = os.getcwd()
    jobdir = os.path.join(here, jobname)
    pipelinefile = os.path.join(here, "pipeline.cfg")
    sys.path.append(jobdir)
    if debug:
        # show us DEBUG logging
        sys.argv.append("-d")
    else:
        # show us INFO logging
        sys.argv.append("-v")


def celery_cmd(args):
    from celery.bin import celery
    base = celery.CeleryCommand(app='tkp.distribute.celery.tasks.trap')
    base.execute_from_commandline(sys.argv[1:])


def run_job(args):
    print "running job '%s'" % args.name
{'fraction_non_alphanumeric': 0.0950956937799043, 'fraction_numerical': 0.0005980861244019139, 'mean_word_length': 2.2492105902356085, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5241541', 'n_tokens_mistral': 3994, 'n_tokens_neox': 3802, 'n_words': 1245}