<commit_before># A post is considered to be extracted from an existing file, and always
# consists in a title, followed by a line of '------------', followed
# immediatly by the tags with a leading #. It is a list of strings, where empty
# ones were removed.
# If it is a valid post, it has at minima 4 lines (title, line of -, tags,
# date)
# They should be extracted recursively, and fed to the different routines
# `_from_post`.
# Note that all doctests are commented, and now executed in the py.test suite,
# for compatibility reasons between py2.7 and py3.3
from __future__ import unicode_literals
from datetime import date
from collections import Counter
from collections import OrderedDict as od
import re
import io


def is_valid_post(post):
    """
    Check that it has all the required arguments

    If the post is valid, the function returns True. Otherwise, an ValueError
    is raised with a description of the problem.

    #>>> good = ["Toto", "-------", "# non-linear, pk", "*21/12/2012*"]
    #>>> is_valid_post(good)
    #True

    #>>> is_valid_post(["Toto", "-------", "*21/12/2012*"])
    #Traceback (most recent call last):
        #...
    #ValueError: Post contains under four lines
    #>>> is_valid_post(["Toto", "-----", "# something", "12/12/042*"])
    #Traceback (most recent call last):
        #...
    #ValueError: The date could not be read
    #>>> is_valid_post(['Toto', '=======', '# something', '*21/12/2012*'])
    #Traceback (most recent call last):
        #...
    #ValueError: Post does not contain dashes
    #>>> is_valid_post(['', '-------', '# non-linear, pk', '*21/12/2012*'])
    #Traceback (most recent call last):
        #...
    #ValueError: Post title is empty
    #>>> is_valid_post(['Toto', '-------', '*21/12/2012*', 'something'])
    #Traceback (most recent call last):
        #...
    #ValueError: Tags were not found after the dashes
    """
    if len(post) < 4:
        raise ValueError("Post contains under four lines")
    else:
        # Recover the index of the line of dashes, in case of long titles
        index = 0
        dashes = [e for e in post if e.find('----') != -1]
        try:
            index = post.index(dashes[0])
        except IndexError:
            raise ValueError("Post does not contain dashes")
        if index:
            if not post[0]:
                raise ValueError("Post title is empty")
            if post[index+1] and post[index+1].find('#') == -1:
                raise ValueError("Tags were not found after the dashes")
            if post[index+2]:
                match = re.match(
                    r"\*[0-9]{2}/[0-1][1-9]/[0-9]{4}\*",
                    post[index+2])
                if match is None:
                    raise ValueError("The date could not be read")
    return True


def extract_tags_from_post(post):
    """
    Recover the tags from an extracted post

    .. note::

        No tests are being done to ensure that the third line exists, as only
        valid posts, determined with :func:`is_valid_post` are sent to this
        routine.

    #>>> post = ['Toto', '-------', ' # non-linear, pk', '*21/12/2012*']
    #>>> extract_tags_from_post(post)
    #(['non-linear', 'pk'], ['Toto', '-------', '*21/12/2012*'])
    """
    tag_line = post[2].strip()
    if tag_line and tag_line[0] == '#':
        tags = [elem.strip() for elem in tag_line[1:].split(',')]

    return tags, post[:2]+post[3:]


def extract_title_from_post(post):
    """
    Recover the title from an extracted post

    #>>> post = ["Toto", "-------", "# non-linear, pk", "*21/12/2012*"]
    #>>> extract_title_from_post(post)
    #'Toto'
    """
    return post[0]


def extract_date_from_post(post):
    """
    Recover the date from an extracted post, and return the correct post

    #>>> post = ["Toto", "-------", "*21/12/2012*", "Something something"]
    #>>> extract_date_from_post(post)
    #(datetime.date(2012, 12, 21), ['Toto', '-------', 'Something something'])
    #>>> extract_date_from_post(["Toto", "---------", "meh"])
    #Traceback (most recent call last):
        #...
    #ValueError: No date found in the post
    """
    for index, line in enumerate(post):
        match = re.match(
            r"\*([0-9]{2})/([0-1][0-9])/([0-9]{4})\*",
            line)
        if match:
            assert len(match.groups()) == 3
            day, month, year = match.groups()
            extracted_date = date(int(year), int(month), int(day))
            return extracted_date, post[:index]+post[index+1:]
    raise ValueError("No date found in the post")


def normalize_post(post):
    """
    If a title has several lines, merge them

    #>>> post = ['Toto', 'has a long title', '-------', '# bla', '*08/11/2010*']
    #>>> normalize_post(post)
    #['Toto has a long title', '-------', '# bla', '*08/11/2010*']
    #>>> is_valid_post(_)
    #True
    """
    # Remove trailing \n
    post = [line.rstrip('\n') for line in post]

    # Recover the dashline (title of the post)
    dashes = [e for e in post if e.find('----') != -1]
    dashline_index = post.index(dashes[0])

    title = ' '.join([post[index] for index in range(dashline_index)])
    normalized_post = [title]
    normalized_post.extend(post[dashline_index:])

    return normalized_post


def extract_corpus_from_post(post):
    """
    Recover the whole content of a post

    #>>> post = ["Toto", "-------", "# non-linear, pk", "*21/12/2012*",
    #...         "This morning I woke", "", "up and it was a nice weather"]
    #>>> extract_corpus_from_post(post)
    #['This morning I woke', '', 'up and it was a nice weather']
    """
    return post[4:]


def extract_title_and_posts_from_text(text):
    """
    From an entire text (array), recover each posts and the file's title

    """
    # Make a first pass to recover the title (first line that is underlined
    # with = signs) and the indices of the dash, that signals a new post.
    post_starting_indices = []
    for index, line in enumerate(text):
        # Remove white lines at the beginning
        if not line.strip():
            continue
        if line.find("====") != -1:
            title = ' '.join(text[:index]).strip()
        if line.find('----') != -1 and line[0] == '-':
            # find the latest non empty line
            for backward_index in range(1, 10):
                if not text[index-backward_index].strip():
                    post_starting_indices.append(index-backward_index+1)
                    break
    number_of_posts = len(post_starting_indices)

    # Create post_indices such that it stores all the post, so the starting and
    # ending index of each post
    post_indices = []
    for index, elem in enumerate(post_starting_indices):
        if index < number_of_posts - 1:
            post_indices.append([elem, post_starting_indices[index+1]])
        else:
            post_indices.append([elem, len(text)])

    posts = []
    for elem in post_indices:
        start, end = elem
        posts.append(text[start:end])

    # Normalize them all
    for index, post in enumerate(posts):
        posts[index] = normalize_post(post)
        assert is_valid_post(posts[index]) is True

    return title, posts


def from_notes_to_markdown(path, input_tags=None):<commit_msg>final fix for #2, proper error message for missing title<commit_after># A post is considered to be extracted from an existing file, and always
# consists in a title, followed by a line of '------------', followed
# immediatly by the tags with a leading #. It is a list of strings, where empty
# ones were removed.
# If it is a valid post, it has at minima 4 lines (title, line of -, tags,
# date)
# They should be extracted recursively, and fed to the different routines
# `_from_post`.
# Note that all doctests are commented, and now executed in the py.test suite,
# for compatibility reasons between py2.7 and py3.3
from __future__ import unicode_literals
from datetime import date
from collections import Counter
from collections import OrderedDict as od
import re
import io


def is_valid_post(post):
    """
    Check that it has all the required arguments

    If the post is valid, the function returns True. Otherwise, an ValueError
    is raised with a description of the problem.

    #>>> good = ["Toto", "-------", "# non-linear, pk", "*21/12/2012*"]
    #>>> is_valid_post(good)
    #True

    #>>> is_valid_post(["Toto", "-------", "*21/12/2012*"])
    #Traceback (most recent call last):
        #...
    #ValueError: Post contains under four lines
    #>>> is_valid_post(["Toto", "-----", "# something", "12/12/042*"])
    #Traceback (most recent call last):
        #...
    #ValueError: The date could not be read
    #>>> is_valid_post(['Toto', '=======', '# something', '*21/12/2012*'])
    #Traceback (most recent call last):
        #...
    #ValueError: Post does not contain dashes
    #>>> is_valid_post(['', '-------', '# non-linear, pk', '*21/12/2012*'])
    #Traceback (most recent call last):
        #...
    #ValueError: Post title is empty
    #>>> is_valid_post(['Toto', '-------', '*21/12/2012*', 'something'])
    #Traceback (most recent call last):
        #...
    #ValueError: Tags were not found after the dashes
    """
    if len(post) < 4:
        raise ValueError("Post contains under four lines")
    else:
        # Recover the index of the line of dashes, in case of long titles
        index = 0
        dashes = [e for e in post if e.find('----') != -1]
        try:
            index = post.index(dashes[0])
        except IndexError:
            raise ValueError("Post does not contain dashes")
        if index:
            if not post[0]:
                raise ValueError("Post title is empty")
            if post[index+1] and post[index+1].find('#') == -1:
                raise ValueError("Tags were not found after the dashes")
            if post[index+2]:
                match = re.match(
                    r"\*[0-9]{2}/[0-1][1-9]/[0-9]{4}\*",
                    post[index+2])
                if match is None:
                    raise ValueError("The date could not be read")
    return True


def extract_tags_from_post(post):
    """
    Recover the tags from an extracted post

    .. note::

        No tests are being done to ensure that the third line exists, as only
        valid posts, determined with :func:`is_valid_post` are sent to this
        routine.

    #>>> post = ['Toto', '-------', ' # non-linear, pk', '*21/12/2012*']
    #>>> extract_tags_from_post(post)
    #(['non-linear', 'pk'], ['Toto', '-------', '*21/12/2012*'])
    """
    tag_line = post[2].strip()
    if tag_line and tag_line[0] == '#':
        tags = [elem.strip() for elem in tag_line[1:].split(',')]

    return tags, post[:2]+post[3:]


def extract_title_from_post(post):
    """
    Recover the title from an extracted post

    #>>> post = ["Toto", "-------", "# non-linear, pk", "*21/12/2012*"]
    #>>> extract_title_from_post(post)
    #'Toto'
    """
    return post[0]


def extract_date_from_post(post):
    """
    Recover the date from an extracted post, and return the correct post

    #>>> post = ["Toto", "-------", "*21/12/2012*", "Something something"]
    #>>> extract_date_from_post(post)
    #(datetime.date(2012, 12, 21), ['Toto', '-------', 'Something something'])
    #>>> extract_date_from_post(["Toto", "---------", "meh"])
    #Traceback (most recent call last):
        #...
    #ValueError: No date found in the post
    """
    for index, line in enumerate(post):
        match = re.match(
            r"\*([0-9]{2})/([0-1][0-9])/([0-9]{4})\*",
            line)
        if match:
            assert len(match.groups()) == 3
            day, month, year = match.groups()
            extracted_date = date(int(year), int(month), int(day))
            return extracted_date, post[:index]+post[index+1:]
    raise ValueError("No date found in the post")


def normalize_post(post):
    """
    If a title has several lines, merge them

    #>>> post = ['Toto', 'has a long title', '-------', '# bla', '*08/11/2010*']
    #>>> normalize_post(post)
    #['Toto has a long title', '-------', '# bla', '*08/11/2010*']
    #>>> is_valid_post(_)
    #True
    """
    # Remove trailing \n
    post = [line.rstrip('\n') for line in post]

    # Recover the dashline (title of the post)
    dashes = [e for e in post if e.find('----') != -1]
    dashline_index = post.index(dashes[0])

    title = ' '.join([post[index] for index in range(dashline_index)])
    normalized_post = [title]
    normalized_post.extend(post[dashline_index:])

    return normalized_post


def extract_corpus_from_post(post):
    """
    Recover the whole content of a post

    #>>> post = ["Toto", "-------", "# non-linear, pk", "*21/12/2012*",
    #...         "This morning I woke", "", "up and it was a nice weather"]
    #>>> extract_corpus_from_post(post)
    #['This morning I woke', '', 'up and it was a nice weather']
    """
    return post[4:]


def extract_title_and_posts_from_text(text):
    """
    From an entire text (array), recover each posts and the file's title

    """
    # Make a first pass to recover the title (first line that is underlined
    # with = signs) and the indices of the dash, that signals a new post.
    post_starting_indices = []
    has_title = False
    for index, line in enumerate(text):
        # Remove white lines at the beginning
        if not line.strip():
            continue
        if line.find("====") != -1:
            title = ' '.join(text[:index]).strip()
            has_title = True
        if line.find('----') != -1 and line[0] == '-':
            # find the latest non empty line
            for backward_index in range(1, 10):
                if not text[index-backward_index].strip():
                    post_starting_indices.append(index-backward_index+1)
                    break

    if not has_title:
        raise ValueError("You should specify a title to your file"
                         ", underlined with = signs")
    number_of_posts = len(post_starting_indices)

    # Create post_indices such that it stores all the post, so the starting and
    # ending index of each post
    post_indices = []
    for index, elem in enumerate(post_starting_indices):
        if index < number_of_posts - 1:
            post_indices.append([elem, post_starting_indices[index+1]])
        else:
            post_indices.append([elem, len(text)])

    posts = []
    for elem in post_indices:
        start, end = elem
        posts.append(text[start:end])

    # Normalize them all
    for index, post in enumerate(posts):
        posts[index] = normalize_post(post)
        assert is_valid_post(posts[index]) is True

    return title, posts


def from_notes_to_markdown(path, input_tags=None):
{'fraction_non_alphanumeric': 0.15557809330628802, 'fraction_numerical': 0.02251521298174442, 'mean_word_length': 2.405710338475708, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5302842', 'n_tokens_mistral': 4717, 'n_tokens_neox': 4447, 'n_words': 1445}