<commit_before># Copyright (c) <NAME>, 2015
# See LICENSE for details.

from __future__ import absolute_import, division, print_function

import os
import textwrap

from collections import OrderedDict

from jinja2 import Template


# Returns a structure like:
#
# OrderedDict([
#   ("",
#    {
#      ("142", "misc"): u"",
#      ("1", "feature"): u"some cool description",
#    }),
#   ("Names", {}),
#   ("Web", {("3", "bugfix"): u"Fixed a thing"}),
# ])
#
# We should really use attrs.
#
# Also returns a list of the paths that the fragments were taken from.
def find_fragments(base_directory, sections, fragment_directory, definitions):
    """
    Sections are a dictonary of section names to paths.
    """
    content = OrderedDict()
    fragment_filenames = []

    for key, val in sections.items():

        if fragment_directory is not None:
            section_dir = os.path.join(base_directory, val, fragment_directory)
        else:
            section_dir = os.path.join(base_directory, val)

        files = os.listdir(section_dir)

        file_content = {}

        for basename in files:
            parts = basename.split(u".")

            if len(parts) == 1:
                continue
            else:
                ticket, category = parts[:2]

            if category not in definitions:
                continue

            full_filename = os.path.join(section_dir, basename)
            fragment_filenames.append(full_filename)
            with open(full_filename, "rb") as f:
                data = f.read().decode("utf8", "replace")
            if (ticket, category) in file_content:
                raise ValueError(
                    "multiple files for {}.{} in {}".format(
                        ticket, category, section_dir
                    )
                )
            file_content[ticket, category] = data

        content[key] = file_content

    return content, fragment_filenames


# Takes the output from find_fragments above. Probably it would be useful to
# add an example output here. Next time someone digs deep enough to figure it
# out, please do so...
def split_fragments(fragments, definitions):

    output = OrderedDict()

    for section_name, section_fragments in fragments.items():
        section = {}

        for (ticket, category), content in section_fragments.items():

            content = textwrap.indent(content.strip(), "  ")[2:]

            if definitions[category]["showcontent"] is False:
                content = u""

            texts = section.get(category, {})

            if texts.get(content):
                texts[content] = sorted(texts[content] + [ticket])
            else:
                texts[content] = [ticket]

            section[category] = texts

        output[section_name] = section

    return output<commit_msg>Provide our own indenting function<commit_after># Copyright (c) <NAME>, 2015
# See LICENSE for details.

from __future__ import absolute_import, division, print_function

import os
import textwrap

from collections import OrderedDict

from jinja2 import Template


# Returns a structure like:
#
# OrderedDict([
#   ("",
#    {
#      ("142", "misc"): u"",
#      ("1", "feature"): u"some cool description",
#    }),
#   ("Names", {}),
#   ("Web", {("3", "bugfix"): u"Fixed a thing"}),
# ])
#
# We should really use attrs.
#
# Also returns a list of the paths that the fragments were taken from.
def find_fragments(base_directory, sections, fragment_directory, definitions):
    """
    Sections are a dictonary of section names to paths.
    """
    content = OrderedDict()
    fragment_filenames = []

    for key, val in sections.items():

        if fragment_directory is not None:
            section_dir = os.path.join(base_directory, val, fragment_directory)
        else:
            section_dir = os.path.join(base_directory, val)

        files = os.listdir(section_dir)

        file_content = {}

        for basename in files:
            parts = basename.split(u".")

            if len(parts) == 1:
                continue
            else:
                ticket, category = parts[:2]

            if category not in definitions:
                continue

            full_filename = os.path.join(section_dir, basename)
            fragment_filenames.append(full_filename)
            with open(full_filename, "rb") as f:
                data = f.read().decode("utf8", "replace")
            if (ticket, category) in file_content:
                raise ValueError(
                    "multiple files for {}.{} in {}".format(
                        ticket, category, section_dir
                    )
                )
            file_content[ticket, category] = data

        content[key] = file_content

    return content, fragment_filenames


def indent(text, prefix):
    """
    Adds `prefix` to the beginning of non-empty lines in `text`.
    """
    # Based on Python 3's textwrap.indent
    def prefixed_lines():
        for line in text.splitlines(True):
            yield (prefix + line if line.strip() else line)
    return u"".join(prefixed_lines())


# Takes the output from find_fragments above. Probably it would be useful to
# add an example output here. Next time someone digs deep enough to figure it
# out, please do so...
def split_fragments(fragments, definitions):

    output = OrderedDict()

    for section_name, section_fragments in fragments.items():
        section = {}

        for (ticket, category), content in section_fragments.items():

            content = indent(content.strip(), u"  ")[2:]

            if definitions[category]["showcontent"] is False:
                content = u""

            texts = section.get(category, {})

            if texts.get(content):
                texts[content] = sorted(texts[content] + [ticket])
            else:
                texts[content] = [ticket]

            section[category] = texts

        output[section_name] = section

    return output
{'fraction_non_alphanumeric': 0.1069931242663089, 'fraction_numerical': 0.004863323830286768, 'mean_word_length': 2.1808, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5687104', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1596, 'n_words': 497}