<commit_before>#!/usr/bin/env python3
"""
This file is part of Mbed TLS (https://tls.mbed.org)

Copyright (c) 2018, Arm Limited, All Rights Reserved

Purpose

This script checks the current state of the source code for minor issues,
including incorrect file permissions, presence of tabs, non-Unix line endings,
trailing whitespace, presence of UTF-8 BOM, and TODO comments.
Note: requires python 3, must be run from Mbed TLS root.
"""

import os
import argparse
import logging
import codecs
import sys


class IssueTracker(object):
    """Base class for issue tracking. Issues should inherit from this and
    overwrite either issue_with_line if they check the file line by line, or
    overwrite check_file_for_issue if they check the file as a whole."""

    def __init__(self):
        self.heading = ""
        self.files_exemptions = []
        self.files_with_issues = {}

    def should_check_file(self, filepath):
        for files_exemption in self.files_exemptions:
            if filepath.endswith(files_exemption):
                return False
        return True

    def issue_with_line(self, line):
        raise NotImplementedError

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            for i, line in enumerate(iter(f.readline, b"")):
                self.check_file_line(filepath, line, i + 1)

    def record_issue(self, filepath, line_number):
        if filepath not in self.files_with_issues.keys():
            self.files_with_issues[filepath] = []
        self.files_with_issues[filepath].append(line_number)

    def check_file_line(self, filepath, line, line_number):
        if self.issue_with_line(line):
            self.record_issue(filepath, line_number)

    def output_file_issues(self, logger):
        if self.files_with_issues.values():
            logger.info(self.heading)
            for filename, lines in sorted(self.files_with_issues.items()):
                if lines:
                    logger.info("{}: {}".format(
                        filename, ", ".join(str(x) for x in lines)
                    ))
                else:
                    logger.info(filename)
            logger.info("")


class PermissionIssueTracker(IssueTracker):
    """Track files with bad permissions.

    Files that are not executable scripts must not be executable."""

    def __init__(self):
        super().__init__()
        self.heading = "Incorrect permissions:"

    def check_file_for_issue(self, filepath):
        if not (os.access(filepath, os.X_OK) ==
                filepath.endswith((".sh", ".pl", ".py"))):
            self.files_with_issues[filepath] = None


class EndOfFileNewlineIssueTracker(IssueTracker):
    """Track files that end with an incomplete line
    (no newline character at the end of the last line)."""

    def __init__(self):
        super().__init__()
        self.heading = "Missing newline at end of file:"

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            if not f.read().endswith(b"\n"):
                self.files_with_issues[filepath] = None


class Utf8BomIssueTracker(IssueTracker):
    """Track files that start with a UTF-8 BOM.
    Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM."""

    def __init__(self):
        super().__init__()
        self.heading = "UTF-8 BOM present:"

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            if f.read().startswith(codecs.BOM_UTF8):
                self.files_with_issues[filepath] = None


class LineEndingIssueTracker(IssueTracker):
    """Track files with non-Unix line endings (i.e. files with CR)."""

    def __init__(self):
        super().__init__()
        self.heading = "Non Unix line endings:"

    def issue_with_line(self, line):
        return b"\r" in line


class TrailingWhitespaceIssueTracker(IssueTracker):
    """Track lines with trailing whitespace."""

    def __init__(self):
        super().__init__()
        self.heading = "Trailing whitespace:"
        self.files_exemptions = [".md"]

    def issue_with_line(self, line):
        return line.rstrip(b"\r\n") != line.rstrip()


class TabIssueTracker(IssueTracker):
    """Track lines with tabs."""

    def __init__(self):
        super().__init__()
        self.heading = "Tabs present:"
        self.files_exemptions = [
            "Makefile", "generate_visualc_files.pl"
        ]

    def issue_with_line(self, line):
        return b"\t" in line


class MergeArtifactIssueTracker(IssueTracker):
    """Track lines with merge artifacts.
    These are leftovers from a ``git merge`` that wasn't fully edited."""

    def __init__(self):
        super().__init__()
        self.heading = "Merge artifact:"

    def issue_with_line(self, filepath, line):
        # Detect leftover git conflict markers.
        if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '):
            return True
        if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3
            return True
        if line.rstrip(b'\r\n') == b'=======' and \
           not filepath.endswith('.md'):
            return True
        return False

    def check_file_line(self, filepath, line, line_number):
        if self.issue_with_line(filepath, line):
            self.record_issue(filepath, line_number)

class TodoIssueTracker(IssueTracker):
    """Track lines containing ``TODO``."""

    def __init__(self):
        super().__init__()
        self.heading = "TODO present:"
        self.files_exemptions = [
            os.path.basename(__file__),
            "benchmark.c",
            "pull_request_template.md",
        ]

    def issue_with_line(self, line):
        return b"todo" in line.lower()


class IntegrityChecker(object):<commit_msg>check-files.py: clean up class structure<commit_after>#!/usr/bin/env python3
"""
This file is part of Mbed TLS (https://tls.mbed.org)

Copyright (c) 2018, Arm Limited, All Rights Reserved

Purpose

This script checks the current state of the source code for minor issues,
including incorrect file permissions, presence of tabs, non-Unix line endings,
trailing whitespace, presence of UTF-8 BOM, and TODO comments.
Note: requires python 3, must be run from Mbed TLS root.
"""

import os
import argparse
import logging
import codecs
import sys


class FileIssueTracker(object):
    """Base class for file-wide issue tracking.

    To implement a checker that processes a file as a whole, inherit from
    this class and implement `check_file_for_issue`.
    """

    def __init__(self):
        self.heading = ""
        self.files_exemptions = []
        self.files_with_issues = {}

    def should_check_file(self, filepath):
        for files_exemption in self.files_exemptions:
            if filepath.endswith(files_exemption):
                return False
        return True

    def check_file_for_issue(self, filepath):
        raise NotImplementedError

    def record_issue(self, filepath, line_number):
        if filepath not in self.files_with_issues.keys():
            self.files_with_issues[filepath] = []
        self.files_with_issues[filepath].append(line_number)

    def output_file_issues(self, logger):
        if self.files_with_issues.values():
            logger.info(self.heading)
            for filename, lines in sorted(self.files_with_issues.items()):
                if lines:
                    logger.info("{}: {}".format(
                        filename, ", ".join(str(x) for x in lines)
                    ))
                else:
                    logger.info(filename)
            logger.info("")

class LineIssueTracker(FileIssueTracker):
    """Base class for line-by-line issue tracking.

    To implement a checker that processes files line by line, inherit from
    this class and implement `line_with_issue`.
    """

    def issue_with_line(self, line, filepath):
        raise NotImplementedError

    def check_file_line(self, filepath, line, line_number):
        if self.issue_with_line(line, filepath):
            self.record_issue(filepath, line_number)

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            for i, line in enumerate(iter(f.readline, b"")):
                self.check_file_line(filepath, line, i + 1)

class PermissionIssueTracker(FileIssueTracker):
    """Track files with bad permissions.

    Files that are not executable scripts must not be executable."""

    def __init__(self):
        super().__init__()
        self.heading = "Incorrect permissions:"

    def check_file_for_issue(self, filepath):
        if not (os.access(filepath, os.X_OK) ==
                filepath.endswith((".sh", ".pl", ".py"))):
            self.files_with_issues[filepath] = None


class EndOfFileNewlineIssueTracker(FileIssueTracker):
    """Track files that end with an incomplete line
    (no newline character at the end of the last line)."""

    def __init__(self):
        super().__init__()
        self.heading = "Missing newline at end of file:"

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            if not f.read().endswith(b"\n"):
                self.files_with_issues[filepath] = None


class Utf8BomIssueTracker(FileIssueTracker):
    """Track files that start with a UTF-8 BOM.
    Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM."""

    def __init__(self):
        super().__init__()
        self.heading = "UTF-8 BOM present:"

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            if f.read().startswith(codecs.BOM_UTF8):
                self.files_with_issues[filepath] = None


class LineEndingIssueTracker(LineIssueTracker):
    """Track files with non-Unix line endings (i.e. files with CR)."""

    def __init__(self):
        super().__init__()
        self.heading = "Non Unix line endings:"

    def issue_with_line(self, line, _filepath):
        return b"\r" in line


class TrailingWhitespaceIssueTracker(LineIssueTracker):
    """Track lines with trailing whitespace."""

    def __init__(self):
        super().__init__()
        self.heading = "Trailing whitespace:"
        self.files_exemptions = [".md"]

    def issue_with_line(self, line, _filepath):
        return line.rstrip(b"\r\n") != line.rstrip()


class TabIssueTracker(LineIssueTracker):
    """Track lines with tabs."""

    def __init__(self):
        super().__init__()
        self.heading = "Tabs present:"
        self.files_exemptions = [
            "Makefile", "generate_visualc_files.pl"
        ]

    def issue_with_line(self, line, _filepath):
        return b"\t" in line


class MergeArtifactIssueTracker(LineIssueTracker):
    """Track lines with merge artifacts.
    These are leftovers from a ``git merge`` that wasn't fully edited."""

    def __init__(self):
        super().__init__()
        self.heading = "Merge artifact:"

    def issue_with_line(self, line, _filepath):
        # Detect leftover git conflict markers.
        if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '):
            return True
        if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3
            return True
        if line.rstrip(b'\r\n') == b'=======' and \
           not _filepath.endswith('.md'):
            return True
        return False

class TodoIssueTracker(LineIssueTracker):
    """Track lines containing ``TODO``."""

    def __init__(self):
        super().__init__()
        self.heading = "TODO present:"
        self.files_exemptions = [
            os.path.basename(__file__),
            "benchmark.c",
            "pull_request_template.md",
        ]

    def issue_with_line(self, line, _filepath):
        return b"todo" in line.lower()


class IntegrityChecker(object):
{'fraction_non_alphanumeric': 0.11048278190411884, 'fraction_numerical': 0.0025320729237002025, 'mean_word_length': 2.6224396209110363, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4526769', 'n_tokens_mistral': 3621, 'n_tokens_neox': 3411, 'n_words': 966}