<commit_before># Copyright 2013 (c) <NAME> <<EMAIL>>

import os
import subprocess
import re
from coveralls import gitrepo


def is_source_file(args,filepath):
    """Returns true if it is a C++ source file."""
    if args.extension:
        return os.path.splitext(filepath)[1] in args.extension
    else:
        return os.path.splitext(filepath)[1] in ['.h', '.hpp', '.cpp', '.cc', '.c']


def exclude_paths(args):
    """Returns the absolute paths for excluded path."""
    results = []
    if args.exclude:
        for excl_path in args.exclude + ['.git', '.svn']:
            results.append(os.path.abspath(os.path.join(args.root, excl_path)))
    return results


def is_under_exclude_path(args, filepath):
    """Returns true if the filepath is under the one of the exclude path
    """
    excl_paths = exclude_paths(args)
    abspath = os.path.abspath(filepath)
    for excluded_path in excl_paths:
        relpath = os.path.relpath(abspath, excluded_path)
        if len(relpath) > 3 and relpath[:3] != '../':
            return True
    return False


def run_gcov(args):
    excl_paths = exclude_paths(args)
    for root, dirs, files in os.walk(args.root):
        filtered_dirs = []
        for dirpath in dirs:
            abspath = os.path.abspath(os.path.join(root, dirpath))
            if not abspath in excl_paths:
                filtered_dirs.append(dirpath)
        dirs[:] = filtered_dirs
        for filepath in files:
            basename, ext = os.path.splitext(filepath)
            if ext == '.gcno':
                if re.search(r".*\.c.*", basename):
                    subprocess.call(
                        'cd %s && %s %s.o' % (root, args.gcov, basename),
                        shell=True)
                else:
                    subprocess.call(
                        'cd %s && %s %s' % (root, args.gcov, basename),
                        shell=True)


def collect(args):
    """Collect coverage reports."""
    excl_paths = exclude_paths(args)

    report = {}
    if args.repo_token:
        report['repo_token'] = args.repo_token
    report['service_name'] = args.service_name
    report['service_job_id'] = args.service_job_id

    discoverd_files = set()
    report['source_files'] = []
    abs_root = os.path.abspath(args.root)
    for root, dirs, files in os.walk(args.root):
        filtered_dirs = []
        for dirpath in dirs:
            abspath = os.path.abspath(os.path.join(root, dirpath))
            if not abspath in excl_paths:
                filtered_dirs.append(dirpath)
        dirs[:] = filtered_dirs

        for filepath in files:
            if os.path.splitext(filepath)[1] == '.gcov':
                gcov_path = os.path.join(os.path.join(root, filepath))
                with open(gcov_path) as fobj:
                    source_file_line = fobj.readline()
                    source_file_path = source_file_line.split(':')[-1].strip()
                    if not os.path.isabs(source_file_path):
                        source_file_path = os.path.abspath(
                            os.path.join(root, source_file_path))
                    src_path = os.path.relpath(source_file_path, abs_root)
                    if len(src_path) > 3 and src_path[:3] == '../':
                        continue
                    if is_under_exclude_path(args, source_file_path):
                        continue

                    src_report = {}
                    src_report['name'] = src_path
                    discoverd_files.add(src_path)
                    with open(src_path) as src_file:
                        src_report['source'] = src_file.read()

                    coverage = []
                    for line in fobj:
                        report_fields = line.split(':')
                        cov_num = report_fields[0].strip()
                        line_num = int(report_fields[1].strip())
                        text = report_fields[2]
                        if line_num == 0:
                            continue
                        if cov_num == '-':
                            coverage.append(None)
                        elif cov_num == '#####':
                            # Avoid false positives.
                            if (text.lstrip().startswith('static') or
                                    text.strip() == '}'):
                                coverage.append(None)
                            else:
                                coverage.append(0)
                        elif cov_num == '=====':
                            # This is indicitive of a gcov output parse
                            # error.
                            coverage.append(0)
                        else:
                            coverage.append(int(cov_num))
                src_report['coverage'] = coverage
                report['source_files'].append(src_report)

    # Also collects the source files that have no coverage reports.
    for root, dirs, files in os.walk(args.root):
        filtered_dirs = []
        for dirpath in dirs:
            abspath = os.path.abspath(os.path.join(root, dirpath))
            if not abspath in excl_paths:
                filtered_dirs.append(dirpath)
        dirs[:] = filtered_dirs

        for filename in files:
            if not is_source_file(args,filename):
                continue
            filepath = os.path.relpath(os.path.join(root, filename), abs_root)
            if not filepath in discoverd_files:
                src_report = {}
                src_report['name'] = filepath
                coverage = []
                with open(filepath) as fobj:
                    for line in fobj:
                        coverage.append(None)
                    fobj.seek(0)
                    src_report['source'] = fobj.read()
                src_report['coverage'] = coverage
                report['source_files'].append(src_report)
    report['git'] = gitrepo.gitrepo('.')
    return report<commit_msg>dont upload un-covered files in excluded path, allow single files in excludes<commit_after># Copyright 2013 (c) <NAME> <<EMAIL>>

import os
import subprocess
import re
from coveralls import gitrepo


def is_source_file(args,filepath):
    """Returns true if it is a C++ source file."""
    if args.extension:
        return os.path.splitext(filepath)[1] in args.extension
    else:
        return os.path.splitext(filepath)[1] in ['.h', '.hpp', '.cpp', '.cc', '.c']


def exclude_paths(args):
    """Returns the absolute paths for excluded path."""
    results = []
    if args.exclude:
        for excl_path in args.exclude + ['.git', '.svn']:
            results.append(os.path.abspath(os.path.join(args.root, excl_path)))
    return results


def is_under_exclude_path(args, filepath):
    """Returns true if the filepath is under the one of the exclude path
    """
    excl_paths = exclude_paths(args)
    abspath = os.path.abspath(filepath)
    for excluded_path in excl_paths:
        if os.path.isdir(excluded_path):
            relpath = os.path.relpath(abspath, excluded_path)
            if len(relpath) > 3 and relpath[:3] != '../':
                return True
        else:
            absexcludefile = os.path.abspath(excluded_path)
            if absexcludefile == abspath:
                return True
    return False


def run_gcov(args):
    excl_paths = exclude_paths(args)
    for root, dirs, files in os.walk(args.root):
        filtered_dirs = []
        for dirpath in dirs:
            abspath = os.path.abspath(os.path.join(root, dirpath))
            if not abspath in excl_paths:
                filtered_dirs.append(dirpath)
        dirs[:] = filtered_dirs
        for filepath in files:
            basename, ext = os.path.splitext(filepath)
            if ext == '.gcno':
                if re.search(r".*\.c.*", basename):
                    subprocess.call(
                        'cd %s && %s %s.o' % (root, args.gcov, basename),
                        shell=True)
                else:
                    subprocess.call(
                        'cd %s && %s %s' % (root, args.gcov, basename),
                        shell=True)


def collect(args):
    """Collect coverage reports."""
    excl_paths = exclude_paths(args)

    report = {}
    if args.repo_token:
        report['repo_token'] = args.repo_token
    report['service_name'] = args.service_name
    report['service_job_id'] = args.service_job_id

    discoverd_files = set()
    report['source_files'] = []
    abs_root = os.path.abspath(args.root)
    for root, dirs, files in os.walk(args.root):
        filtered_dirs = []
        for dirpath in dirs:
            abspath = os.path.abspath(os.path.join(root, dirpath))
            if not abspath in excl_paths:
                filtered_dirs.append(dirpath)
        dirs[:] = filtered_dirs

        for filepath in files:
            if os.path.splitext(filepath)[1] == '.gcov':
                gcov_path = os.path.join(os.path.join(root, filepath))
                with open(gcov_path) as fobj:
                    source_file_line = fobj.readline()
                    source_file_path = source_file_line.split(':')[-1].strip()
                    if not os.path.isabs(source_file_path):
                        source_file_path = os.path.abspath(
                            os.path.join(root, source_file_path))
                    src_path = os.path.relpath(source_file_path, abs_root)
                    if len(src_path) > 3 and src_path[:3] == '../':
                        continue
                    if is_under_exclude_path(args, source_file_path):
                        continue

                    src_report = {}
                    src_report['name'] = src_path
                    discoverd_files.add(src_path)
                    with open(src_path) as src_file:
                        src_report['source'] = src_file.read()

                    coverage = []
                    for line in fobj:
                        report_fields = line.split(':')
                        cov_num = report_fields[0].strip()
                        line_num = int(report_fields[1].strip())
                        text = report_fields[2]
                        if line_num == 0:
                            continue
                        if cov_num == '-':
                            coverage.append(None)
                        elif cov_num == '#####':
                            # Avoid false positives.
                            if (text.lstrip().startswith('static') or
                                    text.strip() == '}'):
                                coverage.append(None)
                            else:
                                coverage.append(0)
                        elif cov_num == '=====':
                            # This is indicitive of a gcov output parse
                            # error.
                            coverage.append(0)
                        else:
                            coverage.append(int(cov_num))
                src_report['coverage'] = coverage
                report['source_files'].append(src_report)

    # Also collects the source files that have no coverage reports.
    for root, dirs, files in os.walk(args.root):
        filtered_dirs = []
        for dirpath in dirs:
            abspath = os.path.abspath(os.path.join(root, dirpath))
            if not abspath in excl_paths:
                filtered_dirs.append(dirpath)
        dirs[:] = filtered_dirs

        for filename in files:
            if not is_source_file(args,filename):
                continue
            if is_under_exclude_path(args,filename):
                continue
            filepath = os.path.relpath(os.path.join(root, filename), abs_root)
            if not filepath in discoverd_files:
                src_report = {}
                src_report['name'] = filepath
                coverage = []
                with open(filepath) as fobj:
                    for line in fobj:
                        coverage.append(None)
                    fobj.seek(0)
                    src_report['source'] = fobj.read()
                src_report['coverage'] = coverage
                report['source_files'].append(src_report)
    report['git'] = gitrepo.gitrepo('.')
    return report
{'fraction_non_alphanumeric': 0.10540034210311966, 'fraction_numerical': 0.003095218701637208, 'mean_word_length': 1.4654618473895582, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2318892', 'n_tokens_mistral': 3502, 'n_tokens_neox': 3376, 'n_words': 763}