<commit_before>#!/usr/bin/env python

import sys
import os
import subprocess as sp
from functools import partial
import shlex
import logging

import yaml
import argh
from argh import arg
import networkx as nx
from networkx.drawing.nx_pydot import write_dot
import pandas


from . import utils
from .build import build_recipes
from . import docker_utils
from . import lint_functions
from . import linting
from . import github_integration

logger = logging.getLogger(__name__)
# NOTE:
#
# A package is the name of the software package, like `bowtie`.
#
# A recipe is the path to the recipe of one version of a package, like
# `recipes/bowtie` or `recipes/bowtie/1.0.1`.


@arg('recipe_folder', help='Path to top-level dir of recipes.')
@arg('config', help='Path to yaml file specifying the configuration')
@arg(
    '--packages',
    nargs="+",
    help='Glob for package[s] to build. Default is to build all packages. Can '
    'be specified more than once')
@arg('--cache', help='''To speed up debugging, use repodata cached locally in
     the provided filename. If the file does not exist, it will be created the
     first time.''')
@arg('--list-funcs', help='''List the linting functions to be used and then
     exit''')
@arg('--only', nargs='+', help='''Only run this linting function. Can be used
     multiple times.''')
@arg('--exclude', nargs='+', help='''Exclude this linting function. Can be used
     multiple times.''')
@arg('--push-status', action='store_true', help='''If set, the lint status will
     be sent to the current commit on github. Also needs --user and --repo to
     be set. Requires the env var GITHUB_TOKEN to be set.''')
@arg('--commit', help='Commit on github on which to update status')
@arg('--user', help='Github user')
@arg('--repo', help='Github repo')
@arg('--git-range', help='Git range', nargs=2)
def lint(recipe_folder, config, packages="*", cache=None, list_funcs=False,
         only=None, exclude=None, push_status=False, user='bioconda',
         commit=None, repo='bioconda-recipes', git_range=None):
    """
    Lint recipes

    If --push-status is not set, reports a TSV of linting results to stdout.
    Otherwise pushes a commit status to the specified commit on github.
    """
    if list_funcs:
        print('\n'.join([i.__name__ for i in lint_functions.registry]))
        sys.exit(0)

    df = linting.channel_dataframe(cache=cache)
    registry = lint_functions.registry

    if only is not None:
        registry = list(filter(lambda x: x.__name__ in only, registry))
        if len(registry) == 0:
            sys.stderr.write('No valid linting functions selected, exiting.\n')
            sys.exit(1)

    # returns recipe_folder/package
    recipes = list(utils.get_recipes(recipe_folder, package=packages))

    if git_range:
        # returns recipe_folder/package/meta.yaml
        modified = utils.modified_recipes(git_range, recipe_folder, full=True)
        if not modified:
            logger.info('No recipe modified according to git, exiting.')
            return
        recipes = [os.path.dirname(f) for f in modified]
        print(recipes)
        logger.info('Recipes modified according to git: {}'.format(' '.join(packages)))

    report = linting.lint(
        recipes,
        config=config,
        df=df,
        registry=registry,
    )

    summarized = pandas.DataFrame(dict(failed_tests=report.groupby('recipe')['check'].agg('unique')))
    if len(summarized) > 0:
        if push_status:
            github_integration.update_status(user, repo, commit, state='error',
                                             context='linting',
                                             description='linting failed, see travis log', target_url=None)
        print(summarized)
        sys.exit(1)
    else:
            update_status(user, repo, commit, state='success', context='linting',
                          description='linting passed', target_url=None)



@arg('recipe_folder', help='Path to top-level dir of recipes.')
@arg('config', help='Path to yaml file specifying the configuration')
@arg(
    '--packages',
    nargs="+",
    help='Glob for package[s] to build. Default is to build all packages. Can '
    'be specified more than once')
@arg('--git-range', nargs=2,
     help='Git range (e.g. commits or something like '
     '"master HEAD"). All recipes modified within this range will be built if '
     'not present in the channel.')
@arg('--testonly', help='Test packages instead of building')
@arg('--force',
     help='Force building the recipe even if it already exists in '<commit_msg>better reporting for failed linting<commit_after>#!/usr/bin/env python

import sys
import os
import subprocess as sp
from functools import partial
import shlex
import logging

import yaml
import argh
from argh import arg
import networkx as nx
from networkx.drawing.nx_pydot import write_dot
import pandas


from . import utils
from .build import build_recipes
from . import docker_utils
from . import lint_functions
from . import linting
from . import github_integration

logger = logging.getLogger(__name__)
# NOTE:
#
# A package is the name of the software package, like `bowtie`.
#
# A recipe is the path to the recipe of one version of a package, like
# `recipes/bowtie` or `recipes/bowtie/1.0.1`.


@arg('recipe_folder', help='Path to top-level dir of recipes.')
@arg('config', help='Path to yaml file specifying the configuration')
@arg(
    '--packages',
    nargs="+",
    help='Glob for package[s] to build. Default is to build all packages. Can '
    'be specified more than once')
@arg('--cache', help='''To speed up debugging, use repodata cached locally in
     the provided filename. If the file does not exist, it will be created the
     first time.''')
@arg('--list-funcs', help='''List the linting functions to be used and then
     exit''')
@arg('--only', nargs='+', help='''Only run this linting function. Can be used
     multiple times.''')
@arg('--exclude', nargs='+', help='''Exclude this linting function. Can be used
     multiple times.''')
@arg('--push-status', action='store_true', help='''If set, the lint status will
     be sent to the current commit on github. Also needs --user and --repo to
     be set. Requires the env var GITHUB_TOKEN to be set.''')
@arg('--commit', help='Commit on github on which to update status')
@arg('--user', help='Github user')
@arg('--repo', help='Github repo')
@arg('--git-range', help='Git range', nargs=2)
def lint(recipe_folder, config, packages="*", cache=None, list_funcs=False,
         only=None, exclude=None, push_status=False, user='bioconda',
         commit=None, repo='bioconda-recipes', git_range=None):
    """
    Lint recipes

    If --push-status is not set, reports a TSV of linting results to stdout.
    Otherwise pushes a commit status to the specified commit on github.
    """
    if list_funcs:
        print('\n'.join([i.__name__ for i in lint_functions.registry]))
        sys.exit(0)

    df = linting.channel_dataframe(cache=cache)
    registry = lint_functions.registry

    if only is not None:
        registry = list(filter(lambda x: x.__name__ in only, registry))
        if len(registry) == 0:
            sys.stderr.write('No valid linting functions selected, exiting.\n')
            sys.exit(1)

    # returns recipe_folder/package
    recipes = list(utils.get_recipes(recipe_folder, package=packages))

    if git_range:
        # returns recipe_folder/package/meta.yaml
        modified = utils.modified_recipes(git_range, recipe_folder, full=True)
        if not modified:
            logger.info('No recipe modified according to git, exiting.')
            return
        recipes = [os.path.dirname(f) for f in modified]
        logger.info('Recipes modified according to git: {}'.format(' '.join(packages)))

    report = linting.lint(
        recipes,
        config=config,
        df=df,
        registry=registry,
    )

    # the returned dataframe is in tidy format; summarize a bit to get a more
    # reasonable log
    summarized = pandas.DataFrame(
        dict(failed_tests=report.groupby('recipe')['check'].agg('unique')))
    if len(summarized) > 0:
        logger.error('The following recipes failed linting:%s\n', summarized)
        if push_status:
            github_integration.update_status(user, repo, commit, state='error',
                                             context='linting',
                                             description='linting failed, see travis log', target_url=None)
        sys.exit(1)
    else:
            update_status(user, repo, commit, state='success', context='linting',
                          description='linting passed', target_url=None)



@arg('recipe_folder', help='Path to top-level dir of recipes.')
@arg('config', help='Path to yaml file specifying the configuration')
@arg(
    '--packages',
    nargs="+",
    help='Glob for package[s] to build. Default is to build all packages. Can '
    'be specified more than once')
@arg('--git-range', nargs=2,
     help='Git range (e.g. commits or something like '
     '"master HEAD"). All recipes modified within this range will be built if '
     'not present in the channel.')
@arg('--testonly', help='Test packages instead of building')
@arg('--force',
     help='Force building the recipe even if it already exists in '
{'fraction_non_alphanumeric': 0.12003000750187547, 'fraction_numerical': 0.002143392991104919, 'mean_word_length': 3.111013215859031, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '313956', 'n_tokens_mistral': 2771, 'n_tokens_neox': 2698, 'n_words': 939}