<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Test runner that will limit the output of tests to a result set relevant for a
particular diff in Git. That is a diff between revisions or a diff between two
branches' tips.

This test runner will work with both Python 2.7 as well as 3.x.
"""

## Created: 23 May 2015 <NAME> <<EMAIL>>
##
## (c) 2015 by Mega Limited, Auckland, New Zealand
##     http://mega.co.nz/
##     Simplified (2-clause) BSD License.
##
## You should have received a copy of the license along with this
## program.
##
## This file is part of the multi-party chat encryption suite.
##
## This code is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

__author__ = '<NAME> <<EMAIL>>'

import argparse
import os
import re
import logging
import tempfile
import subprocess
import collections
import config

PLATFORMS = {'posix': 'posix',
             'nt': 'win32'}
PROJECT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                            os.path.pardir))


def get_git_line_sets(base, target):
    """
    Obtains the Git diff between the base and target to identify the lines that
    contain alterations in the target code. If branch names are given for the
    base or target, the tips of those branches are used.

    :param base: Base branch/commit for the diff.
    :param target: Target branch/commit for the diff.
    :return: A dictionary of changed line numbers. The key in the dictionary is
        the file path, the value is a set of line numbers.
    """
    # Get the Git output for the desired diff.
    logging.info('Extracting relevant lines from Git diff ...')
    command = 'git diff -U0 {} {}'.format(base, target)
    output = subprocess.check_output(command.split())
    diff = output.decode('latin1').split('\n')

    # Hunt down lines of changes for different files.
    file_line_mapping = collections.defaultdict(set)
    current_file = None
    for line in diff:
        if line.startswith('+++'):
            # Line giving target file.
            for_file = line.split()[1]
            current_file = for_file[2:]
        elif line.startswith('@@'):
            # Line giving alteration line range of diff fragment.
            target_lines = line.split()[2].split(',')
            start_line = int(target_lines[0])
            line_range = int(target_lines[1]) if len(target_lines) == 2 else 1
            # Update our lines if we have target lines.
            if line_range > 0:
                file_line_mapping[current_file].update(range(start_line,
                                                             start_line + line_range - 1))

    return file_line_mapping


def reduce_jshint(file_line_mapping, **extra):
    """
    Runs JSHint on the project with the default configured rules. The output
    is reduced to only contain entries from the Git change set.

    :param file_line_mapping: Mapping of files with changed lines (obtained
        `get_git_line_sets()`).
    :param extra: Optional keyword arguments:
        `norules`: If true, omit verbose output of violated rule identifier
                   (default: `False` to include rules).
    :return: A formatted string suitable for output.
    """
    norules = extra['norules'] if 'norules' in extra else False
    # Get the JSHint output.
    logging.info('Obtaining JSHint output ...')
    os.chdir(PROJECT_PATH)
    rules = config.JSHINT_RULES if not norules else ''
    command = config.JSHINT_COMMAND.format(binary=config.JSHINT_BIN,
                                           rules=rules)
    output = None
    try:
        output = subprocess.check_output(command.split())
    except subprocess.CalledProcessError as ex:
        # JSHint found something, so it has returned an error code.
        # But we still want the output in the same fashion.
        output = ex.output
    output = output.decode('utf8').split('\n')

    # Go through output and collect only relevant lines to the result.
    result = ['\nJSHint output:\n==============\n']
    jshint_expression = re.compile(r'(.+): line (\d+), col \d+, .+')
    for line in output:
        parse_result = jshint_expression.findall(line)
        # Check if we've got a relevant line.
        if parse_result:
            file_name, line_no = parse_result[0][0], int(parse_result[0][1])
            # Check if the line is part of our selection list.
            if line_no in file_line_mapping[file_name]:
                result.append(line)

    # Add the number of errors and return in a nicely formatted way.
    result.append('\n{} errors'.format(len(result) - 1))
    return '\n'.join(result)


def reduce_jscs(file_line_mapping, **extra):
    """
    Runs JSHCS on the project with the default configured rules. The output
    is reduced to only contain entries from the Git change set.

    :param file_line_mapping: Mapping of files with changed lines (obtained
        `get_git_line_sets()`).
    :param extra: Optional keyword arguments:
        `norules`: If true, omit verbose output of violated rule identifier
                   (default: `False` to include rules).
    :return: A formatted string suitable for output.
    """
    norules = extra['norules'] if 'norules' in extra else False
    # Get the JSCS output.
    logging.info('Obtaining JSCS output ...')
    os.chdir(PROJECT_PATH)
    rules = config.JSHINT_RULES if not norules else ''
    command = config.JSCS_COMMAND.format(binary=config.JSCS_BIN,
                                         rules=rules)
    output = None
    try:
        output = subprocess.check_output(command.split())
    except subprocess.CalledProcessError as ex:
        # JSCS found something, so it has returned an error code.
        # But we still want the output in the same fashion.
        output = ex.output
    output = output.decode('utf8').split('\n\n')

    # Go through output and collect only relevant lines to the result.
    result = ['\nJSCS output:\n============']
    lines_expression = re.compile(r'^ +(\d+) |.*(?:\n|\r\n?)-', re.MULTILINE)
    file_expression = re.compile(r'^[^\b].* \./(.+) :$', re.MULTILINE)
    for item in output:
        # Do the processing for every block here.
        line_no_candidates = lines_expression.findall(item, re.MULTILINE)
        # Check if we've got a relevant block.
        if line_no_candidates and '' in line_no_candidates:
            line_no = int(line_no_candidates[line_no_candidates.index('') - 1])
            file_name = file_expression.findall(item)[0]
            # Check if the line is part of our selection list.
            if line_no in file_line_mapping[file_name]:
                result.append(item)

    # Add the number of errors and return in a nicely formatted way.
    result.append('\n{} code style errors found.'.format(len(result) - 1))<commit_msg>fix #2358: make file regexp not require a leading ./<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Test runner that will limit the output of tests to a result set relevant for a
particular diff in Git. That is a diff between revisions or a diff between two
branches' tips.

This test runner will work with both Python 2.7 as well as 3.x.
"""

## Created: 23 May 2015 <NAME> <<EMAIL>>
##
## (c) 2015 by Mega Limited, Auckland, New Zealand
##     http://mega.co.nz/
##     Simplified (2-clause) BSD License.
##
## You should have received a copy of the license along with this
## program.
##
## This file is part of the multi-party chat encryption suite.
##
## This code is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

__author__ = '<NAME> <<EMAIL>>'

import argparse
import os
import re
import logging
import tempfile
import subprocess
import collections
import config

PLATFORMS = {'posix': 'posix',
             'nt': 'win32'}
PROJECT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                            os.path.pardir))


def get_git_line_sets(base, target):
    """
    Obtains the Git diff between the base and target to identify the lines that
    contain alterations in the target code. If branch names are given for the
    base or target, the tips of those branches are used.

    :param base: Base branch/commit for the diff.
    :param target: Target branch/commit for the diff.
    :return: A dictionary of changed line numbers. The key in the dictionary is
        the file path, the value is a set of line numbers.
    """
    # Get the Git output for the desired diff.
    logging.info('Extracting relevant lines from Git diff ...')
    command = 'git diff -U0 {} {}'.format(base, target)
    output = subprocess.check_output(command.split())
    diff = output.decode('latin1').split('\n')

    # Hunt down lines of changes for different files.
    file_line_mapping = collections.defaultdict(set)
    current_file = None
    for line in diff:
        if line.startswith('+++'):
            # Line giving target file.
            for_file = line.split()[1]
            current_file = for_file[2:]
        elif line.startswith('@@'):
            # Line giving alteration line range of diff fragment.
            target_lines = line.split()[2].split(',')
            start_line = int(target_lines[0])
            line_range = int(target_lines[1]) if len(target_lines) == 2 else 1
            # Update our lines if we have target lines.
            if line_range > 0:
                file_line_mapping[current_file].update(range(start_line,
                                                             start_line + line_range - 1))

    return file_line_mapping


def reduce_jshint(file_line_mapping, **extra):
    """
    Runs JSHint on the project with the default configured rules. The output
    is reduced to only contain entries from the Git change set.

    :param file_line_mapping: Mapping of files with changed lines (obtained
        `get_git_line_sets()`).
    :param extra: Optional keyword arguments:
        `norules`: If true, omit verbose output of violated rule identifier
                   (default: `False` to include rules).
    :return: A formatted string suitable for output.
    """
    norules = extra['norules'] if 'norules' in extra else False
    # Get the JSHint output.
    logging.info('Obtaining JSHint output ...')
    os.chdir(PROJECT_PATH)
    rules = config.JSHINT_RULES if not norules else ''
    command = config.JSHINT_COMMAND.format(binary=config.JSHINT_BIN,
                                           rules=rules)
    output = None
    try:
        output = subprocess.check_output(command.split())
    except subprocess.CalledProcessError as ex:
        # JSHint found something, so it has returned an error code.
        # But we still want the output in the same fashion.
        output = ex.output
    output = output.decode('utf8').split('\n')

    # Go through output and collect only relevant lines to the result.
    result = ['\nJSHint output:\n==============\n']
    jshint_expression = re.compile(r'(.+): line (\d+), col \d+, .+')
    for line in output:
        parse_result = jshint_expression.findall(line)
        # Check if we've got a relevant line.
        if parse_result:
            file_name, line_no = parse_result[0][0], int(parse_result[0][1])
            # Check if the line is part of our selection list.
            if line_no in file_line_mapping[file_name]:
                result.append(line)

    # Add the number of errors and return in a nicely formatted way.
    result.append('\n{} errors'.format(len(result) - 1))
    return '\n'.join(result)


def reduce_jscs(file_line_mapping, **extra):
    """
    Runs JSHCS on the project with the default configured rules. The output
    is reduced to only contain entries from the Git change set.

    :param file_line_mapping: Mapping of files with changed lines (obtained
        `get_git_line_sets()`).
    :param extra: Optional keyword arguments:
        `norules`: If true, omit verbose output of violated rule identifier
                   (default: `False` to include rules).
    :return: A formatted string suitable for output.
    """
    norules = extra['norules'] if 'norules' in extra else False
    # Get the JSCS output.
    logging.info('Obtaining JSCS output ...')
    os.chdir(PROJECT_PATH)
    rules = config.JSHINT_RULES if not norules else ''
    command = config.JSCS_COMMAND.format(binary=config.JSCS_BIN,
                                         rules=rules)
    output = None
    try:
        output = subprocess.check_output(command.split())
    except subprocess.CalledProcessError as ex:
        # JSCS found something, so it has returned an error code.
        # But we still want the output in the same fashion.
        output = ex.output
    output = output.decode('utf8').split('\n\n')

    # Go through output and collect only relevant lines to the result.
    result = ['\nJSCS output:\n============']
    lines_expression = re.compile(r'^ +(\d+) |.*(?:\n|\r\n?)-', re.MULTILINE)
    file_expression = re.compile(r'^[^\b].* (?:\./)?(.+) :$', re.MULTILINE)
    for item in output:
        # Do the processing for every block here.
        line_no_candidates = lines_expression.findall(item, re.MULTILINE)
        # Check if we've got a relevant block.
        if line_no_candidates and '' in line_no_candidates:
            line_no = int(line_no_candidates[line_no_candidates.index('') - 1])
            file_name = file_expression.findall(item)[0]
            # Check if the line is part of our selection list.
            if line_no in file_line_mapping[file_name]:
                result.append(item)

    # Add the number of errors and return in a nicely formatted way.
    result.append('\n{} code style errors found.'.format(len(result) - 1))
{'fraction_non_alphanumeric': 0.1013484435518577, 'fraction_numerical': 0.005738057667479558, 'mean_word_length': 2.6084368530020705, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470031', 'n_tokens_mistral': 4037, 'n_tokens_neox': 3889, 'n_words': 1434}