<commit_before># -*- coding: utf-8 -*-
import ast
import re
import os.path
import difflib

from py3status.helpers import print_stderr


def modules_directory():
    '''
    Get the core modules directory.
    '''
    return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')


def parse_readme():
    '''
    Crude parsing of modules/README.md
    returns a dict of {<module_name>: <documentation>}
    '''
    name = None
    re_mod = re.compile('^\#\#\# <a name="(?P<name>[a-z_0-9]+)"></a>')
    readme_file = os.path.join(modules_directory(), 'README.md')
    modules_dict = {}
    with open(readme_file) as f:
        for row in f.readlines():
            match = re_mod.match(row)
            if match:
                name = match.group('name')
                modules_dict[name] = []
                continue
            if row.startswith('---'):
                name = None
                continue
            if name:
                modules_dict[name].append(row)
    return modules_dict


def core_module_docstrings(include_core=True, include_user=False, config=None):
    '''
    Get docstrings for all core modules and user ones if requested
    returns a dict of {<module_name>: <docstring>}
    '''
    paths = {}
    docstrings = {}
    if include_core:
        for file in os.listdir(modules_directory()):
            if file.endswith(".py"):
                name = file[:-3]
                if name != '__init__':
                    paths[name] = (os.path.join(modules_directory(), file),
                                   'core')

    if include_user:
        # include user modules
        for include_path in sorted(config['include_paths']):
            include_path = os.path.abspath(include_path) + '/'
            if not os.path.isdir(include_path):
                continue
            for file in sorted(os.listdir(include_path)):
                if not file.endswith('.py'):
                    continue
                name = file[:-3]
                paths[name] = (os.path.join(include_path, file), 'user')
    for name in paths:
        path, module_type = paths[name]
        with open(path) as f:
            module = ast.parse(f.read())
            docstring = ast.get_docstring(module)
            docstring = ['{}\n'.format(d)
                         for d in str(docstring).strip().split('\n')]
            docstrings[name] = docstring + ['\n']
    return docstrings


def create_readme(data):
    '''
    Create README.md text for the given module data.
    '''
    out = ['<a name="top"></a>Modules\n========\n\n']
    # Links
    for module in sorted(data.keys()):
        desc = ''.join(data[module]).strip().split('\n')[0]
        out.append('\n**[{name}](#{name})** — {desc}\n'.format(name=module,
                                                          desc=desc))
    # details
    for module in sorted(data.keys()):
        out.append(
            '\n---\n\n### <a name="{name}"></a>{name}\n\n{details}\n'.format(
                name=module,
                details=''.join(data[module]).strip()))
    return ''.join(out)


def update_docstrings():
    '''
    update the docstring of each module using info in the
    modules/README.md file
    '''
    modules_dict = parse_readme()
    files = {}
    # update modules
    for mod in modules_dict:
        mod_file = os.path.join(modules_directory(), mod + '.py')
        with open(mod_file) as f:
            files[mod] = f.readlines()

    for mod in files:<commit_msg>flake8 fix<commit_after># -*- coding: utf-8 -*-
import ast
import re
import os.path
import difflib

from py3status.helpers import print_stderr


def modules_directory():
    '''
    Get the core modules directory.
    '''
    return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')


def parse_readme():
    '''
    Crude parsing of modules/README.md
    returns a dict of {<module_name>: <documentation>}
    '''
    name = None
    re_mod = re.compile('^\#\#\# <a name="(?P<name>[a-z_0-9]+)"></a>')
    readme_file = os.path.join(modules_directory(), 'README.md')
    modules_dict = {}
    with open(readme_file) as f:
        for row in f.readlines():
            match = re_mod.match(row)
            if match:
                name = match.group('name')
                modules_dict[name] = []
                continue
            if row.startswith('---'):
                name = None
                continue
            if name:
                modules_dict[name].append(row)
    return modules_dict


def core_module_docstrings(include_core=True, include_user=False, config=None):
    '''
    Get docstrings for all core modules and user ones if requested
    returns a dict of {<module_name>: <docstring>}
    '''
    paths = {}
    docstrings = {}
    if include_core:
        for file in os.listdir(modules_directory()):
            if file.endswith(".py"):
                name = file[:-3]
                if name != '__init__':
                    paths[name] = (os.path.join(modules_directory(), file),
                                   'core')

    if include_user:
        # include user modules
        for include_path in sorted(config['include_paths']):
            include_path = os.path.abspath(include_path) + '/'
            if not os.path.isdir(include_path):
                continue
            for file in sorted(os.listdir(include_path)):
                if not file.endswith('.py'):
                    continue
                name = file[:-3]
                paths[name] = (os.path.join(include_path, file), 'user')
    for name in paths:
        path, module_type = paths[name]
        with open(path) as f:
            module = ast.parse(f.read())
            docstring = ast.get_docstring(module)
            docstring = ['{}\n'.format(d)
                         for d in str(docstring).strip().split('\n')]
            docstrings[name] = docstring + ['\n']
    return docstrings


def create_readme(data):
    '''
    Create README.md text for the given module data.
    '''
    out = ['<a name="top"></a>Modules\n========\n\n']
    # Links
    for module in sorted(data.keys()):
        desc = ''.join(data[module]).strip().split('\n')[0]
        out.append('\n**[{name}](#{name})** — {desc}\n'.format(name=module,
                                                               desc=desc))
    # details
    for module in sorted(data.keys()):
        out.append(
            '\n---\n\n### <a name="{name}"></a>{name}\n\n{details}\n'.format(
                name=module,
                details=''.join(data[module]).strip()))
    return ''.join(out)


def update_docstrings():
    '''
    update the docstring of each module using info in the
    modules/README.md file
    '''
    modules_dict = parse_readme()
    files = {}
    # update modules
    for mod in modules_dict:
        mod_file = os.path.join(modules_directory(), mod + '.py')
        with open(mod_file) as f:
            files[mod] = f.readlines()

    for mod in files:
{'fraction_non_alphanumeric': 0.1437160034354423, 'fraction_numerical': 0.00214715144574864, 'mean_word_length': 1.8812371134020618, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4476664', 'n_tokens_mistral': 2116, 'n_tokens_neox': 2053, 'n_words': 484}