<commit_before>"""
Initialize a pipeline run from a pipeline and run it
"""

import time
import datetime

from django.core.management.base import BaseCommand
from apps.pipeline.models import Pipeline


class PipelineDoesNotExist(Exception):
    pass


class Command(BaseCommand):
    help = __doc__

    def __print_status(self, program_run):
        labels = ('PROGRAM', 'TIME', 'TOTAL')

        # data for each row
        rows = []

        # longest string in each column
        cmax = dict()

        for l in labels:
            cmax[l] = len(l)

        prev_seconds = 0

        pgruns = program_run.all_programs()
        for pgr in pgruns:
            pgr_name = str(pgr)
            pgr_total = pgr.run_time()

            # convert time to seconds
            pgr_seconds = 0
            if pgr_total != '-':
                parts = map(int, pgr_total.split(':'))
                parts = map(lambda a, b: a * b, parts, [3600, 60, 1])
                pgr_seconds = reduce(lambda a, b: a + b, parts, pgr_seconds)

            # find delta from previous program's time
            pgr_delta = '-'
            if pgr_seconds > prev_seconds or (pgr.completed and not pgr.is_error):
                pgr_delta = str(datetime.timedelta(seconds=pgr_seconds - prev_seconds))

            prev_seconds = pgr_seconds

            # max column lengths
            cmax['PROGRAM'] = max(cmax['PROGRAM'], len(pgr_name))
            cmax['TIME'] = max(cmax['TIME'], len(pgr_delta))
            cmax['TOTAL'] = max(cmax['TOTAL'], len(pgr_total))

            rows.append((pgr_name, pgr_delta, pgr_total))

        # template widths defined by longest output per column
        template = ' '.join(['{' + str(i) + ':%d}' for i in range(len(labels))])
        template = template % tuple(cmax[l] + 1 for l in labels)

        # print it all out
        print(template.format(*labels))
        for row in rows:
            print(template.format(*row))
        print('')

    def add_arguments(self, parser):
            parser.add_argument(
                'pipeline_name',
                type=str,
                help='Name of the pipeline to test'
            )

            parser.add_argument(
                'update_rate',
                type=float,
                help='Rate to update status of program runs'
            )

    def handle(self, **kwargs):
        if 'update_rate' not in kwargs:
            update_rate = 1
        else:
            update_rate = kwargs['update_rate']

        pname = kwargs['pipeline_name']
        ps = Pipeline.objects.filter(name=pname)

        # TODO make exception more descriptive
        if not ps:
            raise PipelineDoesNotExist

        for pipeline in ps:
            prun = pipeline.run(for_test=True)

            # TODO see if this can update based on when program runs finish,
            # not on a clock
            # TODO more descriptive output
            while True:
                if prun.update_all():
                    self.__print_status(prun)

                    break

                self.__print_status(prun)

                # stop as soon as an error is found
                if prun.programs.filter(is_error=True).count() != 0:
                    break

                time.sleep(update_rate)<commit_msg>Improves printout by clearing screen<commit_after>"""
Initialize a pipeline run from a pipeline and run it
"""

import sys
import time
import datetime

from django.core.management.base import BaseCommand
from apps.pipeline.models import Pipeline


class PipelineDoesNotExist(Exception):
    pass


class Command(BaseCommand):
    help = __doc__

    def __print_status(self, program_run):
        labels = ('PROGRAM', 'TIME', 'TOTAL')

        # data for each row
        rows = []

        # longest string in each column
        cmax = dict()

        for l in labels:
            cmax[l] = len(l)

        prev_seconds = 0

        pgruns = program_run.all_programs()
        for pgr in pgruns:
            pgr_name = str(pgr)
            pgr_total = pgr.run_time()

            # convert time to seconds
            pgr_seconds = 0
            if pgr_total != '-':
                parts = map(int, pgr_total.split(':'))
                parts = map(lambda a, b: a * b, parts, [3600, 60, 1])
                pgr_seconds = reduce(lambda a, b: a + b, parts, pgr_seconds)

            # find delta from previous program's time
            pgr_delta = '-'
            if pgr_seconds > prev_seconds or (pgr.completed and not pgr.is_error):
                pgr_delta = str(datetime.timedelta(seconds=pgr_seconds - prev_seconds))

            prev_seconds = pgr_seconds

            # max column lengths
            cmax['PROGRAM'] = max(cmax['PROGRAM'], len(pgr_name))
            cmax['TIME'] = max(cmax['TIME'], len(pgr_delta))
            cmax['TOTAL'] = max(cmax['TOTAL'], len(pgr_total))

            rows.append((pgr_name, pgr_delta, pgr_total))

        # template widths defined by longest output per column
        template = ' '.join(['{' + str(i) + ':%d}' for i in range(len(labels))])
        template = template % tuple(cmax[l] + 1 for l in labels)

        # print it all out
        # clears screen and puts cursor at top left
        sys.stdout.write('\033[2J\033[1;1H')
        print(template.format(*labels))
        for row in rows:
            print(template.format(*row))
        print('')

    def add_arguments(self, parser):
            parser.add_argument(
                'pipeline_name',
                type=str,
                help='Name of the pipeline to test'
            )

            parser.add_argument(
                'update_rate',
                type=float,
                help='Rate to update status of program runs'
            )

    def handle(self, **kwargs):
        if 'update_rate' not in kwargs:
            update_rate = 1
        else:
            update_rate = kwargs['update_rate']

        pname = kwargs['pipeline_name']
        ps = Pipeline.objects.filter(name=pname)

        # TODO make exception more descriptive
        if not ps:
            raise PipelineDoesNotExist

        for pipeline in ps:
            prun = pipeline.run(for_test=True)

            # TODO see if this can update based on when program runs finish,
            # not on a clock
            while True:
                if prun.update_all():
                    self.__print_status(prun)

                    break

                self.__print_status(prun)

                # stop as soon as an error is found
                if prun.programs.filter(is_error=True).count() != 0:
                    break

                time.sleep(update_rate)
{'fraction_non_alphanumeric': 0.09550224887556222, 'fraction_numerical': 0.004947526236881559, 'mean_word_length': 1.7228571428571429, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6636', 'n_tokens_mistral': 1970, 'n_tokens_neox': 1847, 'n_words': 532}