<commit_before># coding: utf-8
import re
import codecs
import copy
import itertools
import os
import tempfile
import shutil
import envoy
import lxml.html

from operator import itemgetter
from collections import defaultdict
from functools import wraps
from pkg_resources import parse_version
from rq.timeouts import JobTimeoutException

from allmychanges.crawler import _extract_version, _extract_date
from allmychanges.utils import get_change_type, strip_long_text, first
from allmychanges.env import Environment
from django.conf import settings
from twiggy_goodies.threading import log


CHANGELOG_LIKE_FILENAMES = ('change', 'release', 'news', 'history')
EXTENSIONS_TO_CHECK = {'.rst', '.md', '.markdown', '.txt', '.htm', '.html'}


def filename_looks_like_a_changelog(filename):
    filename = filename.lower()
    return any((item in filename)
               for item in CHANGELOG_LIKE_FILENAMES)


def compare_versions(left, right):
    return left.date < right.date \
        and left.version < right.version


def parse_changelog(raw_changelog):
    # TODO: похоже этот код нигде не используется
    chunks = raw_changelog.get_chunks()
    versions = itertools.chain(*[chunk.get_versions()
                                 for chunk in chunks])
    versions = sorted(versions, compare_versions)
    return versions


##############################
## NEW
##############################

def node_tostring(node, with_tail=False):
    """Convert's html node to string if it is not a string yet.
    """
    if isinstance(node, basestring):
        return node
    return lxml.html.tostring(node, with_tail=with_tail)



# TODO: move to utils
def strip_outer_tag(text):
    get_children = lxml.html.etree.XPath("text()|*")

    def rec(items):
        if len(items) == 1:
            if isinstance(items[0], basestring):
                return items[0]
            return rec(get_children(items[0]))
        else:
            return u''.join(
                map(node_tostring,
                    items))

    items = lxml.html.fragments_fromstring(text)
    return rec(items)


def get_files(env, walk=os.walk):
    """
    Uses: env.ignore_list, env.search_list and env.dirname.
    """

    ignore_list = env.ignore_list
    search_list = env.search_list

    def in_ignore_list(filename):
        for ignore_prefix in ignore_list:
            if filename.startswith(ignore_prefix):
                return True
        return False

    def search_in_search_list(filename):
        for prefix, markup in search_list:
            if filename.startswith(prefix):
                return True, markup
        return False, None

    for root, dirs, files in walk(env.dirname):
        for filename in files:
            full_filename = os.path.join(root, filename)
            rel_filename = os.path.relpath(full_filename, env.dirname)

            low_filename = rel_filename.lower()
            _, ext = os.path.splitext(low_filename)

            if ext not in EXTENSIONS_TO_CHECK \
               and not filename_looks_like_a_changelog(low_filename):
                continue

            if not in_ignore_list(rel_filename):
                attrs = dict(type='filename',
                             filename=full_filename)

                if search_list:
                    found, markup = search_in_search_list(rel_filename)
                    if found:
                        if markup:
                            attrs['markup'] = markup
                        yield env.push(**attrs)

                else:
                    yield env.push(**attrs)


# TODO: remove
def read_files(root, filenames):
    for filename in filenames:
        with codecs.open(filename, 'r', 'utf-8') as f:
            try:
                content = create_file(
                    os.path.relpath(filename, root),
                    f.read())
            except Exception:
                continue

            yield content


def read_file(obj):
    with codecs.open(obj.filename, 'r', 'utf-8') as f:
        try:
            yield obj.push(
                type='file_content',
                filename=os.path.relpath(obj.filename, obj.dirname),
                content=f.read())
        except Exception:
            pass


# TODO: remove
def parse_files(file_objects):
    """Outputs separate sections (header + notes/items)
    from every file.
    """
    for obj in file_objects:
        markup = get_markup(obj)
        if markup is not None:
            versions = globals().get('parse_{0}_file'.format(markup))(obj)
            for version in versions:
                # this information will be required late
                version['filename'] = get_file_name(obj)
                yield version

def parse_file(env):
    """Outputs separate sections (header + notes/items)
    from every file.
    """
    markup = getattr(env, 'markup', None)

    if not markup:
        markup = get_markup(env.filename, env.content)

    if markup is not None:
        parser = globals().get('parse_{0}_file'.format(markup))
        versions = parser(env)
        for version in versions:
            yield version



def parse_markdown_file(obj):
    import markdown2
    html = markdown2.markdown(obj.content)
    return parse_html_file(obj.push(content=html))


def parse_plain_file(obj):
    from allmychanges.crawler import _parse_item, _starts_with_ident
    current_title = None
    # here we'll track each line distance from corresponding
    # line with version number
    current_sections = []
    current_section = None
    current_ident = None

    for line in obj.content.split('\n'):
        # skip lines like
        # ===================
        if line and line == line[0] * len(line):
            continue

        is_item, ident, text = _parse_item(line)
        version = _extract_version(line)

        # we have to check if item is not version
        # because of python-redis changelog format
        # where versions is unordered list with sublists
        if is_item and not version:
            if not isinstance(current_section, list):
                # wow, a new changelog item list was found
                if current_section is not None:
                    current_sections.append(current_section)
                current_section = []

            current_ident = ident
            current_section.append(text)
        else:
            if version is not None:
                # we found a possible version number, lets
                # start collecting the changes!
                if current_section:
                    current_sections.append(current_section)

                if current_title and current_sections:
                    yield obj.push(type='file_section',
                                   title=current_title,
                                   content=current_sections)

                current_title = line
                current_section = None
                current_sections = []
                current_ident = None

            elif _starts_with_ident(line, current_ident) and isinstance(current_section, list):
                # previous changelog item has continuation on the
                # next line
                current_section[-1] += '\n' + line[current_ident:]
            else:
                # if this is not item, then this is a note
                if current_title:
                    if isinstance(current_section, list):
                        # if there is items in the current section
                        # and we found another plaintext part,
                        # then start another section
                        current_sections.append(current_section)
                        current_section = None

                    if current_section is None:
                        if line:
                            current_section = line
                    else:
                        current_section += u'\n' + line

    if current_section:
        current_sections.append(current_section)

    if current_title and current_sections:
        # if current_section:
        #     current_sections.append(current_section)
        yield obj.push(type='file_section',
                       title=current_title,
                       content=current_sections)



def search_conf_py(root_dir, doc_filename):
    parts = doc_filename.split('/')
    while parts:
        parts = parts[:-1]
        filename = os.path.join(
            root_dir, '/'.join(parts), 'conf.py')
        if os.path.exists(filename):
            return filename


def parse_rst_file(obj):
    with log.fields(filename=obj.filename):
        dirname = os.path.dirname(obj.filename)

        path_key = 'rst_builder_temp_path:' + dirname
        path = obj.cache.get(path_key)
        if path is None:
            path = tempfile.mkdtemp(dir=obj.cache['tmp-dir'])
            obj.cache[path_key] = path

        def create_conf_py():
            filename = os.path.join(path, 'conf.py')
            if os.path.exists(filename):
                os.unlink(filename)
            with codecs.open(filename, 'a', 'utf-8') as f:
                f.write("""
html_theme = 'epub'
html_theme_options = {
    'relbar1': 'false',
    'footer': 'false',
}
master_doc = 'index'
source_suffix = '.rst'
""")

        def copy_conf_py():
            filename = os.path.join(path, 'conf.py')

            # if there is a config for sphinx
            # then use it!
            log.info('Copying conf.py')
            shutil.copy(conf_py, filename)

            # also, use all directories starting from underscore
            # for django it is _ext and _themes, for other
            # projects it maky work or not
            # if it does not work for some important projects, then
            # we'll need a more wise algorithm which will try
            # to figure out conf.py dependencies

            conf_py_dir = os.path.dirname(conf_py)
            for name in os.listdir(conf_py_dir):
                fullname = os.path.join(conf_py_dir, name)
                if name.startswith('_') and os.path.isdir(fullname):
                    destination = os.path.join(path, name)
                    if not os.path.exists(destination):
                        shutil.copytree(fullname, destination)

            with codecs.open(os.path.join(path, 'conf.py'), 'a', 'utf-8') as f:
                f.write("master_doc = 'index'\n")
                f.write("source_suffix = '.rst'\n")


        def generate_html():
            """Tries to generate HTML from reST with current config.
            If operation was successful, returns content of HTML file.
            """
            with codecs.open(os.path.join(path, 'index.rst'), 'w', 'utf-8') as f:
                f.write(obj.content)

            envoy.run('rm -fr {0}/output/index.html'.format(path))
            envoy.run('sphinx-build -b html {0} {0}/output'.format(path))

            output_filename = os.path.join(path, 'output', 'index.html')
            if os.path.exists(output_filename):
                with codecs.open(output_filename, 'r', 'utf-8') as f:
                    return f.read()


        conf_py_key = 'rst_conf_py:' + dirname
        conf_py = obj.cache.get(conf_py_key)

        if conf_py is None:
            conf_py = search_conf_py(obj.dirname, obj.filename)
            if not conf_py:
                conf_py = 'fake-conf.py'
                log.info('Will use fake conf.py')

        html = None
        if conf_py != 'fake-conf.py':
            copy_conf_py()
            html = generate_html()
        if html is None:
            create_conf_py()
            if conf_py != 'fake-conf.py':
                log.info('Attempt to use package\'s conf.py has failed')
            conf_py = 'fake-conf.py'
            html = generate_html()

        if html is None:
            log.info('Unable to parse this rst file')
            html = ''

        obj.cache[conf_py_key] = conf_py
        return parse_html_file(obj.push(content=html))


# TODO: remove
def create_section(title, content=[], version=None, date=None):
    """Each section has a title and a list of content objects.
    Each content object is either a text or a list object.
    """
    result = dict(title=title, content=content)
    if version:
        result['version'] = version
    result['date'] = date
    return result


get_section_title = itemgetter('title')
get_section_content = itemgetter('content')


def parse_html_file(obj):
    """
    Uses fields `content` and `filename` to generate new file_sections.
    """
    parsed = lxml.html.document_fromstring(obj.content)
    headers = [tag for tag in parsed.iter()
               if tag.tag in ('h1', 'h2', 'h3', 'h4')]


    def create_list(el):
        def inner_html(el):
            text = lxml.html.tostring(el).strip()
            text = strip_outer_tag(text)
            return text

        return map(inner_html, el.getchildren())

    def create_notes(children):
        """ This function returns content of the current
        version. It is a list where each item either a HTML
        fragment or a list of html fragments.
        If it is a list of html fragments, then each item
        SHOULD NOT be wrapped into a node like <li> or <p>.
        """
        current_text = ''
        for el in children:
            if el.tag == 'ul':
                if current_text:
                    yield current_text.strip()
                    current_text = ''
                yield create_list(el)
            else:
                current_text += lxml.html.tostring(el)

        if current_text:
            yield current_text.strip()

    def create_full_content(children):
        """Just serialize all childrens and join result."""
        strings = map(lxml.html.tostring, children)
        return u''.join(strings)


    headers = [(header.tag, header.text_content(), header.itersiblings())
               for header in headers]


    def is_header_tag(ch):
        if isinstance(ch.tag, basestring):
            return not ch.tag.startswith('h') or ch.tag > tag


    def process_header(parent, tag, text, all_children):
        children = list(itertools.takewhile(
            is_header_tag,
            all_children))
        sections = list(create_notes(children))

        full_content = create_full_content(children)
        return parent.push(type='file_section',
                           title=text.strip(),
                           content=sections,
                           full_content=full_content)

    # use whole document and filename as a section
    root = process_header(obj, 'h0', obj.filename,
                          parsed.find('body').iterchildren())
    yield root

    # and then process all other items
    for tag, text, all_children in headers:
        yield process_header(root, tag, text, all_children)


def create_file(name, content):
    return dict(name=name, content=content)

get_file_name = itemgetter('name')
get_file_content = itemgetter('content')


def get_markup(filename, content):
    filename = filename.lower()
    content_head = content[:1000].lower().strip()

    if filename.endswith('.rst'):
        return 'rst'

    if filename.endswith('.md'):
        return 'markdown'

    if filename.endswith('.html') or filename.endswith('.htm'):
        return 'html'

    if ':func:`' in content \
       or re.search('`[^` ]+`_', content, re.MULTILINE) is not None \
       or re.search('``[^` ]+``', content, re.MULTILINE) is not None:
        return 'rst'

    if re.search('^[=-]{3,}', content, re.MULTILINE) is not None \
       or re.search('^#{2,}', content, re.MULTILINE) is not None \
       or re.search('\[.*?\]\(.*\)', content, re.MULTILINE) is not None \
       or re.search('\[.*?\]\[.*\]', content, re.MULTILINE) is not None:
        return 'markdown'

    if content_head.startswith('<!doctype ') \
       or content_head.startswith('<html'):
        return 'html'

    return 'plain'


def filter_versions(sections):
    """Searches parts of the files which looks like
    changelog pieces.
    """
    for section in sections:
        version = _extract_version(get_section_title(section))

        if version:
            new_section = copy.deepcopy(section)
            new_section['version'] = version
            yield new_section


def filter_version(section):
    """Searches parts of the files which looks like
    changelog pieces.
    """
    version = _extract_version(section.title)
    if version:
        yield section.push(type='almost_version',
                           version=version)


def extract_metadata(version):
    """Tries to extract date and list items' type.
    """
    def _all_dates(iterable):
        for item in iterable:
            date = _extract_date(item)
            if date:
                yield date

    def _all_list_items():
        for content_part in version.content:
            if isinstance(content_part, list):
                for item in content_part:
                    yield item
            elif isinstance(content_part, basestring):
                yield content_part

    def mention_unreleased(text):
        # here we limit our scoupe of searching
        # unreleased keywords
        # because if keyword is somewhere far from
        # the beginning, than probably it is unrelated
        # to the version itself
        text = text[:300]

        keywords = ('unreleased', 'under development',
                    'not yet released',
                    'release date to be decided')
        lowered = text.lower()
        for keyword in keywords:
            if keyword in lowered:
                return True
        return False

    all_dates = list(_all_dates(itertools.chain([version.title],
                                                _all_list_items())))

    new_version = version.push(type='prerender_items')

    if all_dates:
        new_version.date = first(all_dates)

    if mention_unreleased(version.title):<commit_msg>Now we search release date only in first 200 symbols of the version's description.<commit_after># coding: utf-8
import re
import codecs
import copy
import os
import tempfile
import shutil
import envoy
import lxml.html

from operator import itemgetter
from collections import defaultdict
from functools import wraps
from itertools import islice, chain, takewhile
from pkg_resources import parse_version
from rq.timeouts import JobTimeoutException

from allmychanges.crawler import _extract_version, _extract_date
from allmychanges.utils import get_change_type, strip_long_text, first
from allmychanges.env import Environment
from django.conf import settings
from twiggy_goodies.threading import log


CHANGELOG_LIKE_FILENAMES = ('change', 'release', 'news', 'history')
EXTENSIONS_TO_CHECK = {'.rst', '.md', '.markdown', '.txt', '.htm', '.html'}


def filename_looks_like_a_changelog(filename):
    filename = filename.lower()
    return any((item in filename)
               for item in CHANGELOG_LIKE_FILENAMES)


def compare_versions(left, right):
    return left.date < right.date \
        and left.version < right.version


def parse_changelog(raw_changelog):
    # TODO: похоже этот код нигде не используется
    chunks = raw_changelog.get_chunks()
    versions = chain(*[chunk.get_versions()
                       for chunk in chunks])
    versions = sorted(versions, compare_versions)
    return versions


##############################
## NEW
##############################

def node_tostring(node, with_tail=False):
    """Convert's html node to string if it is not a string yet.
    """
    if isinstance(node, basestring):
        return node
    return lxml.html.tostring(node, with_tail=with_tail)



# TODO: move to utils
def strip_outer_tag(text):
    get_children = lxml.html.etree.XPath("text()|*")

    def rec(items):
        if len(items) == 1:
            if isinstance(items[0], basestring):
                return items[0]
            return rec(get_children(items[0]))
        else:
            return u''.join(
                map(node_tostring,
                    items))

    items = lxml.html.fragments_fromstring(text)
    return rec(items)


def get_files(env, walk=os.walk):
    """
    Uses: env.ignore_list, env.search_list and env.dirname.
    """

    ignore_list = env.ignore_list
    search_list = env.search_list

    def in_ignore_list(filename):
        for ignore_prefix in ignore_list:
            if filename.startswith(ignore_prefix):
                return True
        return False

    def search_in_search_list(filename):
        for prefix, markup in search_list:
            if filename.startswith(prefix):
                return True, markup
        return False, None

    for root, dirs, files in walk(env.dirname):
        for filename in files:
            full_filename = os.path.join(root, filename)
            rel_filename = os.path.relpath(full_filename, env.dirname)

            low_filename = rel_filename.lower()
            _, ext = os.path.splitext(low_filename)

            if ext not in EXTENSIONS_TO_CHECK \
               and not filename_looks_like_a_changelog(low_filename):
                continue

            if not in_ignore_list(rel_filename):
                attrs = dict(type='filename',
                             filename=full_filename)

                if search_list:
                    found, markup = search_in_search_list(rel_filename)
                    if found:
                        if markup:
                            attrs['markup'] = markup
                        yield env.push(**attrs)

                else:
                    yield env.push(**attrs)


# TODO: remove
def read_files(root, filenames):
    for filename in filenames:
        with codecs.open(filename, 'r', 'utf-8') as f:
            try:
                content = create_file(
                    os.path.relpath(filename, root),
                    f.read())
            except Exception:
                continue

            yield content


def read_file(obj):
    with codecs.open(obj.filename, 'r', 'utf-8') as f:
        try:
            yield obj.push(
                type='file_content',
                filename=os.path.relpath(obj.filename, obj.dirname),
                content=f.read())
        except Exception:
            pass


# TODO: remove
def parse_files(file_objects):
    """Outputs separate sections (header + notes/items)
    from every file.
    """
    for obj in file_objects:
        markup = get_markup(obj)
        if markup is not None:
            versions = globals().get('parse_{0}_file'.format(markup))(obj)
            for version in versions:
                # this information will be required late
                version['filename'] = get_file_name(obj)
                yield version

def parse_file(env):
    """Outputs separate sections (header + notes/items)
    from every file.
    """
    markup = getattr(env, 'markup', None)

    if not markup:
        markup = get_markup(env.filename, env.content)

    if markup is not None:
        parser = globals().get('parse_{0}_file'.format(markup))
        versions = parser(env)
        for version in versions:
            yield version



def parse_markdown_file(obj):
    import markdown2
    html = markdown2.markdown(obj.content)
    return parse_html_file(obj.push(content=html))


def parse_plain_file(obj):
    from allmychanges.crawler import _parse_item, _starts_with_ident
    current_title = None
    # here we'll track each line distance from corresponding
    # line with version number
    current_sections = []
    current_section = None
    current_ident = None

    for line in obj.content.split('\n'):
        # skip lines like
        # ===================
        if line and line == line[0] * len(line):
            continue

        is_item, ident, text = _parse_item(line)
        version = _extract_version(line)

        # we have to check if item is not version
        # because of python-redis changelog format
        # where versions is unordered list with sublists
        if is_item and not version:
            if not isinstance(current_section, list):
                # wow, a new changelog item list was found
                if current_section is not None:
                    current_sections.append(current_section)
                current_section = []

            current_ident = ident
            current_section.append(text)
        else:
            if version is not None:
                # we found a possible version number, lets
                # start collecting the changes!
                if current_section:
                    current_sections.append(current_section)

                if current_title and current_sections:
                    yield obj.push(type='file_section',
                                   title=current_title,
                                   content=current_sections)

                current_title = line
                current_section = None
                current_sections = []
                current_ident = None

            elif _starts_with_ident(line, current_ident) and isinstance(current_section, list):
                # previous changelog item has continuation on the
                # next line
                current_section[-1] += '\n' + line[current_ident:]
            else:
                # if this is not item, then this is a note
                if current_title:
                    if isinstance(current_section, list):
                        # if there is items in the current section
                        # and we found another plaintext part,
                        # then start another section
                        current_sections.append(current_section)
                        current_section = None

                    if current_section is None:
                        if line:
                            current_section = line
                    else:
                        current_section += u'\n' + line

    if current_section:
        current_sections.append(current_section)

    if current_title and current_sections:
        # if current_section:
        #     current_sections.append(current_section)
        yield obj.push(type='file_section',
                       title=current_title,
                       content=current_sections)



def search_conf_py(root_dir, doc_filename):
    parts = doc_filename.split('/')
    while parts:
        parts = parts[:-1]
        filename = os.path.join(
            root_dir, '/'.join(parts), 'conf.py')
        if os.path.exists(filename):
            return filename


def parse_rst_file(obj):
    with log.fields(filename=obj.filename):
        dirname = os.path.dirname(obj.filename)

        path_key = 'rst_builder_temp_path:' + dirname
        path = obj.cache.get(path_key)
        if path is None:
            path = tempfile.mkdtemp(dir=obj.cache['tmp-dir'])
            obj.cache[path_key] = path

        def create_conf_py():
            filename = os.path.join(path, 'conf.py')
            if os.path.exists(filename):
                os.unlink(filename)
            with codecs.open(filename, 'a', 'utf-8') as f:
                f.write("""
html_theme = 'epub'
html_theme_options = {
    'relbar1': 'false',
    'footer': 'false',
}
master_doc = 'index'
source_suffix = '.rst'
""")

        def copy_conf_py():
            filename = os.path.join(path, 'conf.py')

            # if there is a config for sphinx
            # then use it!
            log.info('Copying conf.py')
            shutil.copy(conf_py, filename)

            # also, use all directories starting from underscore
            # for django it is _ext and _themes, for other
            # projects it maky work or not
            # if it does not work for some important projects, then
            # we'll need a more wise algorithm which will try
            # to figure out conf.py dependencies

            conf_py_dir = os.path.dirname(conf_py)
            for name in os.listdir(conf_py_dir):
                fullname = os.path.join(conf_py_dir, name)
                if name.startswith('_') and os.path.isdir(fullname):
                    destination = os.path.join(path, name)
                    if not os.path.exists(destination):
                        shutil.copytree(fullname, destination)

            with codecs.open(os.path.join(path, 'conf.py'), 'a', 'utf-8') as f:
                f.write("master_doc = 'index'\n")
                f.write("source_suffix = '.rst'\n")


        def generate_html():
            """Tries to generate HTML from reST with current config.
            If operation was successful, returns content of HTML file.
            """
            with codecs.open(os.path.join(path, 'index.rst'), 'w', 'utf-8') as f:
                f.write(obj.content)

            envoy.run('rm -fr {0}/output/index.html'.format(path))
            envoy.run('sphinx-build -b html {0} {0}/output'.format(path))

            output_filename = os.path.join(path, 'output', 'index.html')
            if os.path.exists(output_filename):
                with codecs.open(output_filename, 'r', 'utf-8') as f:
                    return f.read()


        conf_py_key = 'rst_conf_py:' + dirname
        conf_py = obj.cache.get(conf_py_key)

        if conf_py is None:
            conf_py = search_conf_py(obj.dirname, obj.filename)
            if not conf_py:
                conf_py = 'fake-conf.py'
                log.info('Will use fake conf.py')

        html = None
        if conf_py != 'fake-conf.py':
            copy_conf_py()
            html = generate_html()
        if html is None:
            create_conf_py()
            if conf_py != 'fake-conf.py':
                log.info('Attempt to use package\'s conf.py has failed')
            conf_py = 'fake-conf.py'
            html = generate_html()

        if html is None:
            log.info('Unable to parse this rst file')
            html = ''

        obj.cache[conf_py_key] = conf_py
        return parse_html_file(obj.push(content=html))


# TODO: remove
def create_section(title, content=[], version=None, date=None):
    """Each section has a title and a list of content objects.
    Each content object is either a text or a list object.
    """
    result = dict(title=title, content=content)
    if version:
        result['version'] = version
    result['date'] = date
    return result


get_section_title = itemgetter('title')
get_section_content = itemgetter('content')


def parse_html_file(obj):
    """
    Uses fields `content` and `filename` to generate new file_sections.
    """
    parsed = lxml.html.document_fromstring(obj.content)
    headers = [tag for tag in parsed.iter()
               if tag.tag in ('h1', 'h2', 'h3', 'h4')]


    def create_list(el):
        def inner_html(el):
            text = lxml.html.tostring(el).strip()
            text = strip_outer_tag(text)
            return text

        return map(inner_html, el.getchildren())

    def create_notes(children):
        """ This function returns content of the current
        version. It is a list where each item either a HTML
        fragment or a list of html fragments.
        If it is a list of html fragments, then each item
        SHOULD NOT be wrapped into a node like <li> or <p>.
        """
        current_text = ''
        for el in children:
            if el.tag == 'ul':
                if current_text:
                    yield current_text.strip()
                    current_text = ''
                yield create_list(el)
            else:
                current_text += lxml.html.tostring(el)

        if current_text:
            yield current_text.strip()

    def create_full_content(children):
        """Just serialize all childrens and join result."""
        strings = map(lxml.html.tostring, children)
        return u''.join(strings)


    headers = [(header.tag, header.text_content(), header.itersiblings())
               for header in headers]


    def is_header_tag(ch):
        if isinstance(ch.tag, basestring):
            return not ch.tag.startswith('h') or ch.tag > tag


    def process_header(parent, tag, text, all_children):
        children = list(takewhile(
            is_header_tag,
            all_children))
        sections = list(create_notes(children))

        full_content = create_full_content(children)
        return parent.push(type='file_section',
                           title=text.strip(),
                           content=sections,
                           full_content=full_content)

    # use whole document and filename as a section
    root = process_header(obj, 'h0', obj.filename,
                          parsed.find('body').iterchildren())
    yield root

    # and then process all other items
    for tag, text, all_children in headers:
        yield process_header(root, tag, text, all_children)


def create_file(name, content):
    return dict(name=name, content=content)

get_file_name = itemgetter('name')
get_file_content = itemgetter('content')


def get_markup(filename, content):
    filename = filename.lower()
    content_head = content[:1000].lower().strip()

    if filename.endswith('.rst'):
        return 'rst'

    if filename.endswith('.md'):
        return 'markdown'

    if filename.endswith('.html') or filename.endswith('.htm'):
        return 'html'

    if ':func:`' in content \
       or re.search('`[^` ]+`_', content, re.MULTILINE) is not None \
       or re.search('``[^` ]+``', content, re.MULTILINE) is not None:
        return 'rst'

    if re.search('^[=-]{3,}', content, re.MULTILINE) is not None \
       or re.search('^#{2,}', content, re.MULTILINE) is not None \
       or re.search('\[.*?\]\(.*\)', content, re.MULTILINE) is not None \
       or re.search('\[.*?\]\[.*\]', content, re.MULTILINE) is not None:
        return 'markdown'

    if content_head.startswith('<!doctype ') \
       or content_head.startswith('<html'):
        return 'html'

    return 'plain'


def filter_versions(sections):
    """Searches parts of the files which looks like
    changelog pieces.
    """
    for section in sections:
        version = _extract_version(get_section_title(section))

        if version:
            new_section = copy.deepcopy(section)
            new_section['version'] = version
            yield new_section


def filter_version(section):
    """Searches parts of the files which looks like
    changelog pieces.
    """
    version = _extract_version(section.title)
    if version:
        yield section.push(type='almost_version',
                           version=version)


def extract_metadata(version):
    """Tries to extract date and list items' type.
    """
    def _all_dates(iterable):
        # we don't need dates which are deep in the release notes
        for item in islice(iterable, 0, 4):
            # and we search only through first 200 symbols
            # of each part
            date = _extract_date(item[:200])
            if date:
                yield date

    def _all_list_items():
        for content_part in version.content:
            if isinstance(content_part, list):
                for item in content_part:
                    yield item
            elif isinstance(content_part, basestring):
                yield content_part

    def mention_unreleased(text):
        # here we limit our scoupe of searching
        # unreleased keywords
        # because if keyword is somewhere far from
        # the beginning, than probably it is unrelated
        # to the version itself
        text = text[:300]

        keywords = ('unreleased', 'under development',
                    'not yet released',
                    'release date to be decided')
        lowered = text.lower()
        for keyword in keywords:
            if keyword in lowered:
                return True
        return False

    all_dates = list(_all_dates(chain([version.title],
                                      _all_list_items())))

    new_version = version.push(type='prerender_items')

    if all_dates:
        new_version.date = first(all_dates)

    if mention_unreleased(version.title):
{'fraction_non_alphanumeric': 0.09798876808587158, 'fraction_numerical': 0.0022961794893075495, 'mean_word_length': 1.9549579007602387, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6696668', 'n_tokens_mistral': 10266, 'n_tokens_neox': 9754, 'n_words': 2744}