<commit_before>#!/usr/bin/env python

import click
import copy
import os
import popper.utils as pu
import signal
import subprocess
import time
import sys
import re
import requests

from popper.cli import pass_context
from subprocess import check_output
from collections import defaultdict


@click.command('run',
               short_help='Run one or more pipelines and report on their '
                          'status.')
@click.argument('pipeline', required=False)
@click.option(
    '--timeout',
    help='Timeout limit for pipeline. Use s for seconds, m for minutes and h '
         'for hours. A single integer can also be used to specify timeout '
         'in seconds. Use double quotes if you wish to use more than one unit.'
         'For example: --timeout "2m 20s" will mean 140 seconds.',
    required=False,
    show_default=True,
    default="10800s"
)
@click.option(
    '--output',
    help='Directory where logs are stored, relative to pipeline folder.',
    required=False,
    show_default=True,
    default="popper/"
)
@click.option(
    '--skip',
    help="Comma-separated list of stages to skip. Only works when a single"
         "pipeline is being executed.",
    required=False,
)
@click.option(
    '--ignore-errors',
    help='Execute all pipelines even if there is a failure.',
    is_flag=True,
    required=False,
)
@click.option(
    '--requirement-level',
    type=click.Choice(['warn', 'ignore', 'fail']), default='fail',
    help='Determines how to handle missing requirements. warn simply'
         'emits a wanring and continues normally; ignore runs the'
         'specified pipelines despite missing unfulfilled'
         'requirements; fail exits with an error on unfilfilled'
         'requirements.'
)
@click.option(
    '--no-badge-update',
    help='Do not update the status of the pipeline against the badge server.',
    is_flag=True,
    required=False,
)
@pass_context
def cli(ctx, pipeline, timeout, skip, ignore_errors, output,
        no_badge_update, requirement_level):
    """Executes one or more pipelines and reports on their status. When
    PIPELINE is given, it executes only the pipeline with that name. If the
    argument is omitted, all pipelines are executed in lexicographical order.
    """
    project_pipelines = pu.read_config()['pipelines']

    if len(project_pipelines) == 0:
        pu.info("No pipelines defined in .popper.yml. "
                "Run popper init --help for more info.", fg='yellow')
        sys.exit(0)

    project_root = pu.get_project_root()

    cwd = os.getcwd()

    pipelines = get_pipelines_to_execute(cwd, pipeline, project_pipelines)

    if os.environ.get('CI', False):
        pipes_from_log = pipelines_from_commit_message(project_pipelines)
        if len(pipes_from_log) != 0:
            pu.info("Found 'CI', ignoring PIPELINE argument.")
            pipelines = pipes_from_log

    pipelines = {pipe_n: pipe_c for pipe_n, pipe_c in pipelines.items()
                 if check_requirements(pipe_n, pipe_c, requirement_level)}

    pipelines = check_skiplist(pipelines, skip)

    for pipe_n, pipe_d in pipelines.items():
        for env in pipe_d.get('envs', ['host']):
            args = [project_root, pipe_n, pipe_d, env, timeout, skip,
                    ignore_errors, output]
            executions = get_executions_for_pipeline(pipe_d.get('vars'))
            status = run_pipeline(*args, executions=executions)
            if status == 'FAIL' and not ignore_errors:
                break

    if not len(pipelines):
        pu.info("No pipelines to execute")
        sys.exit(0)

    os.chdir(cwd)

    if os.environ.get('CI', False) and not no_badge_update:
        update_badge(status)

    if status == 'FAIL':
        pu.fail("Failed to execute pipeline")


def get_executions_for_pipeline(env_vars):
    executions = []
    if env_vars:
        for env_var in env_vars:
            executions.append(env_var)
    else:
        executions.append("")
    return executions


def set_env_vars(env_vars):
    for env_var in env_vars:
        os.environ[env_var] = env_vars[env_var]


def unset_env_vars(env_vars):
    for env_var in env_vars:
        del os.environ[env_var]


def get_pipelines_to_execute(cwd, pipe_n, project_pipelines):
    pipelines = {}
    if pipe_n:
        if pipe_n not in project_pipelines:
            pu.fail("Pipeline '{}' not in .popper.yml".format(pipe_n))
        return {pipe_n: project_pipelines[pipe_n]}
    else:
        cwd_pipe_n = os.path.basename(cwd)
        if cwd_pipe_n in project_pipelines:
            pipelines = {cwd_pipe_n: project_pipelines[cwd_pipe_n]}
        else:
            pipelines = project_pipelines

    return pipelines


def check_skiplist(pipelines, skiplist):
    """
    Filters a list of pipelines based on a list of pipelines and stages to
    skip.
    """
    if skiplist is None:
        return pipelines

    new_pipelist = {}

    pu.info(skiplist)

    skiplist = set(skiplist.split(','))

    # parse stages to skip in the form of pipe:stage
    stage_skip = defaultdict(set)
    for item in skiplist:
        tup = item.split(':')
        if len(tup) > 1:
            pipe, stage = tup
            stage_skip[pipe].add(stage)

    # only include pipes and stages that aren't in the skip list
    for name, pipeline in pipelines.items():
        if name in skiplist:
            continue
        stages = set(pipeline['stages']) - skiplist - stage_skip[name]

        pipeline = dict(pipeline)
        pipeline['stages'] = stages

        new_pipelist[name] = pipeline

    return new_pipelist

<commit_msg>Guarantee stage order when skipping<commit_after>#!/usr/bin/env python

import click
import copy
import os
import popper.utils as pu
import signal
import subprocess
import time
import sys
import re
import requests

from popper.cli import pass_context
from subprocess import check_output
from collections import defaultdict


@click.command('run',
               short_help='Run one or more pipelines and report on their '
                          'status.')
@click.argument('pipeline', required=False)
@click.option(
    '--timeout',
    help='Timeout limit for pipeline. Use s for seconds, m for minutes and h '
         'for hours. A single integer can also be used to specify timeout '
         'in seconds. Use double quotes if you wish to use more than one unit.'
         'For example: --timeout "2m 20s" will mean 140 seconds.',
    required=False,
    show_default=True,
    default="10800s"
)
@click.option(
    '--output',
    help='Directory where logs are stored, relative to pipeline folder.',
    required=False,
    show_default=True,
    default="popper/"
)
@click.option(
    '--skip',
    help="Comma-separated list of stages to skip. Only works when a single"
         "pipeline is being executed.",
    required=False,
)
@click.option(
    '--ignore-errors',
    help='Execute all pipelines even if there is a failure.',
    is_flag=True,
    required=False,
)
@click.option(
    '--requirement-level',
    type=click.Choice(['warn', 'ignore', 'fail']), default='fail',
    help='Determines how to handle missing requirements. warn simply'
         'emits a wanring and continues normally; ignore runs the'
         'specified pipelines despite missing unfulfilled'
         'requirements; fail exits with an error on unfilfilled'
         'requirements.'
)
@click.option(
    '--no-badge-update',
    help='Do not update the status of the pipeline against the badge server.',
    is_flag=True,
    required=False,
)
@pass_context
def cli(ctx, pipeline, timeout, skip, ignore_errors, output,
        no_badge_update, requirement_level):
    """Executes one or more pipelines and reports on their status. When
    PIPELINE is given, it executes only the pipeline with that name. If the
    argument is omitted, all pipelines are executed in lexicographical order.
    """
    project_pipelines = pu.read_config()['pipelines']

    if len(project_pipelines) == 0:
        pu.info("No pipelines defined in .popper.yml. "
                "Run popper init --help for more info.", fg='yellow')
        sys.exit(0)

    project_root = pu.get_project_root()

    cwd = os.getcwd()

    pipelines = get_pipelines_to_execute(cwd, pipeline, project_pipelines)

    if os.environ.get('CI', False):
        pipes_from_log = pipelines_from_commit_message(project_pipelines)
        if len(pipes_from_log) != 0:
            pu.info("Found 'CI', ignoring PIPELINE argument.")
            pipelines = pipes_from_log

    pipelines = {pipe_n: pipe_c for pipe_n, pipe_c in pipelines.items()
                 if check_requirements(pipe_n, pipe_c, requirement_level)}

    pipelines = check_skiplist(pipelines, skip)

    for pipe_n, pipe_d in pipelines.items():
        for env in pipe_d.get('envs', ['host']):
            args = [project_root, pipe_n, pipe_d, env, timeout, skip,
                    ignore_errors, output]
            executions = get_executions_for_pipeline(pipe_d.get('vars'))
            status = run_pipeline(*args, executions=executions)
            if status == 'FAIL' and not ignore_errors:
                break

    if not len(pipelines):
        pu.info("No pipelines to execute")
        sys.exit(0)

    os.chdir(cwd)

    if os.environ.get('CI', False) and not no_badge_update:
        update_badge(status)

    if status == 'FAIL':
        pu.fail("Failed to execute pipeline")


def get_executions_for_pipeline(env_vars):
    executions = []
    if env_vars:
        for env_var in env_vars:
            executions.append(env_var)
    else:
        executions.append("")
    return executions


def set_env_vars(env_vars):
    for env_var in env_vars:
        os.environ[env_var] = env_vars[env_var]


def unset_env_vars(env_vars):
    for env_var in env_vars:
        del os.environ[env_var]


def get_pipelines_to_execute(cwd, pipe_n, project_pipelines):
    pipelines = {}
    if pipe_n:
        if pipe_n not in project_pipelines:
            pu.fail("Pipeline '{}' not in .popper.yml".format(pipe_n))
        return {pipe_n: project_pipelines[pipe_n]}
    else:
        cwd_pipe_n = os.path.basename(cwd)
        if cwd_pipe_n in project_pipelines:
            pipelines = {cwd_pipe_n: project_pipelines[cwd_pipe_n]}
        else:
            pipelines = project_pipelines

    return pipelines


def check_skiplist(pipelines, skiplist):
    """
    Filters a list of pipelines based on a list of pipelines and stages to
    skip.
    """
    if skiplist is None:
        return pipelines

    new_pipelist = {}

    pu.info(skiplist)

    skiplist = set(skiplist.split(','))

    # parse stages to skip in the form of pipe:stage
    stage_skip = defaultdict(set)
    for item in skiplist:
        tup = item.split(':')
        if len(tup) > 1:
            pipe, stage_n = tup
            stage_skip[pipe].add(stage_n)

    # only include pipes and stages that aren't in the skip list
    for name, pipeline in pipelines.items():
        if name in skiplist:
            continue
        stages = [stage for stage in pipeline['stages']
                  if stage not in skiplist and stage not in stage_skip[name]]

        pipeline = dict(pipeline)
        pipeline['stages'] = stages

        new_pipelist[name] = pipeline

    return new_pipelist


{'fraction_non_alphanumeric': 0.0958455133315617, 'fraction_numerical': 0.0028346177695101425, 'mean_word_length': 2.727302740178277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '99434', 'n_tokens_mistral': 3540, 'n_tokens_neox': 3387, 'n_words': 1024}