<commit_before>#!/usr/bin/env python

'''
show_info.py -- show source sizes over time
'''

import os
import re
import sys
from collections import namedtuple

from colorama import Back, Fore, Style
from git import Repo
from natsort import natsorted

class Project:
    config = None
    def parse_yaml(self, obj):
        self.config = yaml.load(obj)

    def is_interesting_item(self, item):
        def is_interesting_name(name):
            return os.path.splitext(name)[-1] in ['.c', '.py']
        return (item.type == 'blob' and
        # is_interesting_path(item.path) and
        is_interesting_name(item.name))



# TODO make configurable
def is_interesting_path(path):
    if re.compile(r'^(docs|examples|scripts|tests|__init)/').match(path):
        return False
    if re.compile('/testsuite/').search(path):
        return False
    return True

def is_interesting_item(item):
    return (item.type == 'blob' and
        is_interesting_path(item.path) and
        is_interesting_name(item.name))
# TODO make configurable
def is_interesting_name(name):
    return os.path.splitext(name)[-1] in ['.c', '.py']


# TODO make configurable
def is_interesting_path(path):
    if re.compile(r'^(docs|examples|scripts|tests|__init)/').match(path):
        return False
    if re.compile('/testsuite/').search(path):
        return False
    return True

def is_interesting_item(item):
    return (item.type == 'blob' and
        is_interesting_path(item.path) and
        is_interesting_name(item.name))

def find_sources(tree):
    source_items = tree.traverse(predicate=lambda item,_: is_interesting_item(item))
    return source_items

def find_sources2(tree):
    proj = Project()
    proj.config = dict(path_extensions=['.c', '.py'])<commit_msg>wip: compile dull patterns into project<commit_after>#!/usr/bin/env python

'''
show_info.py -- show source sizes over time
'''

import os
import re
import sys
from collections import namedtuple

from colorama import Back, Fore, Style
from git import Repo
from natsort import natsorted

class Project:
    config = None
    def parse_yaml(self, obj):
        self.config = yaml.load(obj)
    def compile(self):
        dull_words = '|'.join(self.config['dull_words'])
        self.dull_regex = re.compile(r'\b(' + dull_words + r')\b')

    def is_interesting_item(self, item):
        def is_interesting_name(name):
            return os.path.splitext(name)[-1] in self.config['source_extensions']
        def is_interesting_path(path):
            return not self.dull_regex.search(path)
        return (item.type == 'blob' and
            is_interesting_path(item.path) and
            is_interesting_name(item.name))



# TODO make configurable
def is_interesting_path(path):
    if re.compile(r'^(docs|examples|scripts|tests|__init)/').match(path):
        return False
    if re.compile('/testsuite/').search(path):
        return False
    return True

def is_interesting_item(item):
    return (item.type == 'blob' and
        is_interesting_path(item.path) and
        is_interesting_name(item.name))
# TODO make configurable
def is_interesting_name(name):
    return os.path.splitext(name)[-1] in ['.c', '.py']


# TODO make configurable
def is_interesting_path(path):
    if re.compile(r'^(docs|examples|scripts|tests|__init)/').match(path):
        return False
    if re.compile('/testsuite/').search(path):
        return False
    return True

def is_interesting_item(item):
    return (item.type == 'blob' and
        is_interesting_path(item.path) and
        is_interesting_name(item.name))

def find_sources(tree):
    source_items = tree.traverse(predicate=lambda item,_: is_interesting_item(item))
    return source_items

def find_sources2(tree):
    proj = Project()
    proj.config = dict(
        dull_words=['docs','examples', 'scripts', 'testsuite', 'tests', '__init__.py'],
        source_extensions=['.c', '.py'])
    proj.compile()
{'fraction_non_alphanumeric': 0.12471249680552006, 'fraction_numerical': 0.001533350370559673, 'mean_word_length': 3.4833906071019474, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330669', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1215, 'n_words': 285}