<commit_before>#!/usr/bin/env python
# coding: utf-8
"""
    diff-coverage

    This module will, in a somewhat inflexible way, compare a diff coverage.py
    data to determine whether lines added or modified in the diff, were executed
    during a coverage session.

    requires http://python-patch.googlecode.com/svn/trunk/patch.py
    which is included in this package with attribution
"""
from collections import defaultdict
from optparse import OptionParser
import logging
import os
import string
import sys

import coverage

import patch
import settings


TEMPLATE_FOLDER = os.path.abspath(os.path.join(__file__, '..'))
LAYOUT_TEMPLATE_FILE = os.path.join(TEMPLATE_FOLDER, 'templates/layout.html')
ROW_TEMPLATE_FILE = os.path.join(TEMPLATE_FOLDER, 'templates/row.html')
ADDED_LINE = '+'
REMOVED_LINE = '-'
ROOT_PATH = os.getcwd()
COVERAGE_FILE_PATH = os.path.join(ROOT_PATH, settings.COVERAGE_PATH)
coverage_html_dir = os.path.join(os.getcwd(), settings.OUTPUT_COVERAGE_DOC)
line_end = '(?:\n|\r\n?)'
BORDER_STYLE = 'style="border: 1px solid"'

patch_logger = logging.getLogger('patch')
patch_logger.addHandler(logging.NullHandler())


class FileTemplate(string.Template):
    def __init__(self, file_name):
        with open(file_name, 'r') as template_file:
            super(FileTemplate, self).__init__(template_file.read())


def is_ignored_file(file_path):
    for ignored_portion in settings.IGNORED_NAME_PORTIONS:
        try:
            result = bool(ignored_portion.match(file_path))
        except AttributeError:
            result = ignored_portion in file_path

        if result:
            return True

    return False


def get_jenkins_path(file_name):
    file_name_parts = file_name.split('/')
    if len(file_name_parts) > 1:
        file_name_parts = ['_'.join(file_name_parts[:2])] + file_name_parts[2:]

    file_name_parts[-1].replace('.py', '_py')
    return os.path.sep.join(file_name_parts)


def parse_patch(patch_file):
    """returns a dictionary of {filepath:[lines patched]}"""
    patch_set = patch.fromfile(patch_file)
    target_files = set()
    for changed_file in patch_set.items:
        relative_path = changed_file.target
        if not is_ignored_file(relative_path):
            absolute_file_path = os.path.join(ROOT_PATH, relative_path)
            if (os.path.exists(absolute_file_path)
                    and not os.path.isdir(absolute_file_path)):
                target_files.add(absolute_file_path)

    target_lines = defaultdict(list)
    for p in patch_set.items:
        source_file = os.path.join(ROOT_PATH, p.target)
        if source_file not in target_files:
            continue<commit_msg>master: fixed assignment problem<commit_after>#!/usr/bin/env python
# coding: utf-8
"""
    diff-coverage

    This module will, in a somewhat inflexible way, compare a diff coverage.py
    data to determine whether lines added or modified in the diff, were executed
    during a coverage session.

    requires http://python-patch.googlecode.com/svn/trunk/patch.py
    which is included in this package with attribution
"""
from collections import defaultdict
from optparse import OptionParser
import logging
import os
import string
import sys

import coverage

import patch
import settings


TEMPLATE_FOLDER = os.path.abspath(os.path.join(__file__, '..'))
LAYOUT_TEMPLATE_FILE = os.path.join(TEMPLATE_FOLDER, 'templates/layout.html')
ROW_TEMPLATE_FILE = os.path.join(TEMPLATE_FOLDER, 'templates/row.html')
ADDED_LINE = '+'
REMOVED_LINE = '-'
ROOT_PATH = os.getcwd()
COVERAGE_FILE_PATH = os.path.join(ROOT_PATH, settings.COVERAGE_PATH)
coverage_html_dir = os.path.join(os.getcwd(), settings.OUTPUT_COVERAGE_DOC)
line_end = '(?:\n|\r\n?)'
BORDER_STYLE = 'style="border: 1px solid"'

patch_logger = logging.getLogger('patch')
patch_logger.addHandler(logging.NullHandler())


class FileTemplate(string.Template):
    def __init__(self, file_name):
        with open(file_name, 'r') as template_file:
            super(FileTemplate, self).__init__(template_file.read())


def is_ignored_file(file_path):
    for ignored_portion in settings.IGNORED_NAME_PORTIONS:
        try:
            result = bool(ignored_portion.match(file_path))
        except AttributeError:
            result = ignored_portion in file_path

        if result:
            return True

    return False


def get_jenkins_path(file_name):
    file_name_parts = file_name.split('/')
    if len(file_name_parts) > 1:
        file_name_parts = ['_'.join(file_name_parts[:2])] + file_name_parts[2:]

    file_name_parts[-1] = file_name_parts[-1].replace('.py', '_py')
    return os.path.sep.join(file_name_parts)


def parse_patch(patch_file):
    """returns a dictionary of {filepath:[lines patched]}"""
    patch_set = patch.fromfile(patch_file)
    target_files = set()
    for changed_file in patch_set.items:
        relative_path = changed_file.target
        if not is_ignored_file(relative_path):
            absolute_file_path = os.path.join(ROOT_PATH, relative_path)
            if (os.path.exists(absolute_file_path)
                    and not os.path.isdir(absolute_file_path)):
                target_files.add(absolute_file_path)

    target_lines = defaultdict(list)
    for p in patch_set.items:
        source_file = os.path.join(ROOT_PATH, p.target)
        if source_file not in target_files:
            continue
{'fraction_non_alphanumeric': 0.10240626748740907, 'fraction_numerical': 0.0024249207237455697, 'mean_word_length': 3.6504770164787512, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1726601', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1695, 'n_words': 373}