<commit_before>from fnmatch import filter as fn_filter
from os import walk, path
import hashlib

from utils import detect_mimetype
from github import GithubAPIManager
from languages import *


class Pygemony(object):
    def __init__(self, user=None, token=None, owner=None, repo=None):
        # todo_found contains a list of the following layout:
        # ['file_path', 'line_number', 'todo_message', 'md5 of todo']
        self.blacklist = ['build', '.git']
        self.todo_found = []
        self.github = GithubAPIManager(user, token, owner, repo)
        # TODO(ian): Add support for parsing more than one file type
        self.language = self.lookup_language()

    def find_end_comment(self, f):
        # TODO(ian): Remove this function as we no longer support multiline TODO
        todo_content = []
        x = f
        count = 0
        for line in x.readlines():
            todo_content.append(line)
            if self.language.multi_comment[1] in line:
                return todo_content

            if count > 20:
                return None

            todo_content.append(line)
            count += 1

    def _sanitize_todo_line(self, lines):
        # We're mainly aiming to remove newlines and tab characters here.
        lines = lines.replace('\n', '')
        while '    ' in lines or '\t' in lines:
            lines = lines.replace('    ', '')
        for lang in self.language:
            lines = lines.replace(lang.single_comment, '')
        return lines

    @staticmethod
    def hash_todo(todo_content, file_name):
        m = hashlib.md5()
        m.update('{}-{}'.format(todo_content, file_name))
        return str(m.hexdigest())

    def parse_for_todo(self, f, file_):
        for i, line in enumerate(f.readlines()):
            if "TODO" in line and self._starts_with_comment(line):
                line = self._sanitize_todo_line(line)
                self.todo_found.append([file_, i, line, self.hash_todo(line, file_)])

    def parse_by_extension(self, files):
        for lang in self.language:
            for ext in lang.file_exts:
                for file_ in fn_filter(files, ext):
                    yield file_

    def find_all_files(self, root):
        files_found = []

        for roots, _, files in walk(root):
            base_dir = roots.split('/')[1]<commit_msg>Fix py2.6 format string issue<commit_after>from fnmatch import filter as fn_filter
from os import walk, path
import hashlib

from utils import detect_mimetype
from github import GithubAPIManager
from languages import *


class Pygemony(object):
    def __init__(self, user=None, token=None, owner=None, repo=None):
        # todo_found contains a list of the following layout:
        # ['file_path', 'line_number', 'todo_message', 'md5 of todo']
        self.blacklist = ['build', '.git']
        self.todo_found = []
        self.github = GithubAPIManager(user, token, owner, repo)
        # TODO(ian): Add support for parsing more than one file type
        self.language = self.lookup_language()

    def find_end_comment(self, f):
        # TODO(ian): Remove this function as we no longer support multiline TODO
        todo_content = []
        x = f
        count = 0
        for line in x.readlines():
            todo_content.append(line)
            if self.language.multi_comment[1] in line:
                return todo_content

            if count > 20:
                return None

            todo_content.append(line)
            count += 1

    def _sanitize_todo_line(self, lines):
        # We're mainly aiming to remove newlines and tab characters here.
        lines = lines.replace('\n', '')
        while '    ' in lines or '\t' in lines:
            lines = lines.replace('    ', '')
        for lang in self.language:
            lines = lines.replace(lang.single_comment, '')
        return lines

    @staticmethod
    def hash_todo(todo_content, file_name):
        m = hashlib.md5()
        m.update('{0}-{1}'.format(todo_content, file_name))
        return str(m.hexdigest())

    def parse_for_todo(self, f, file_):
        for i, line in enumerate(f.readlines()):
            if "TODO" in line and self._starts_with_comment(line):
                line = self._sanitize_todo_line(line)
                self.todo_found.append([file_, i, line, self.hash_todo(line, file_)])

    def parse_by_extension(self, files):
        for lang in self.language:
            for ext in lang.file_exts:
                for file_ in fn_filter(files, ext):
                    yield file_

    def find_all_files(self, root):
        files_found = []

        for roots, _, files in walk(root):
            base_dir = roots.split('/')[1]
{'fraction_non_alphanumeric': 0.10055508112724168, 'fraction_numerical': 0.004269854824935952, 'mean_word_length': 2.21771978021978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3898032', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1363, 'n_words': 399}