<commit_before># -*- coding: UTF-8 -*-
#
# Copyright (c) 2009-2010 <NAME>, Itaapy, Pierlis, Talend.
#
# Authors: <NAME> <<EMAIL>>
#          <NAME> <<EMAIL>>
#          <NAME> <<EMAIL>>
#
# This file is part of Lpod (see: http://lpod-project.org).
# Lpod is free software; you can redistribute it and/or modify it under
# the terms of either:
#
# a) the GNU General Public License as published by the Free Software
#    Foundation, either version 3 of the License, or (at your option)
#    any later version.
#    Lpod is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#    You should have received a copy of the GNU General Public License
#    along with Lpod.  If not, see <http://www.gnu.org/licenses/>.
#
# b) the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#    http://www.apache.org/licenses/LICENSE-2.0
#

# Import from the Standard Library
from re import compile, escape

# Import from lpod
from bookmark import odf_create_bookmark, odf_create_bookmark_start
from bookmark import odf_create_bookmark_end
from element import FIRST_CHILD, odf_text
from element import register_element_class, odf_element, odf_create_element
from note import odf_create_note, odf_create_annotation
from style import odf_style


def _get_formatted_text(element, context, with_text=True):
    document = context['document']
    rst_mode = context['rst_mode']

    result = []
    if with_text:
        objects = element.xpath('*|text()')
    else:
        objects = element.get_children()
    for obj in objects:
        if type(obj) is odf_text:
            result.append(obj)
        else:
            tag = obj.get_tag()
            # Good tags with text
            if tag in ('text:a', 'text:p'):
                result.append(_get_formatted_text(obj, context,
                                                 with_text=True))
            # Try to convert some styles in rst_mode
            elif tag == 'text:span':
                # XXX Move this part in span.py ??
                text = _get_formatted_text(obj, context, with_text=True)
                if not rst_mode:
                    result.append(text)
                    continue
                if not text.strip():
                    result.append(text)
                    continue
                style = obj.get_style()
                if style is None:
                    result.append(text)
                    continue
                style = document.get_style("text", style)
                properties = style.get_properties()
                if properties is None:
                    continue
                # Compute before, text and after
                before = ''
                for c in text:
                    if c.isspace():
                        before += c
                    else:
                        break
                after = ''
                for c in reversed(text):
                    if c.isspace():
                        after = c + after
                    else:
                        break
                text = text.strip()
                # Bold ?
                if properties.get('fo:font-weight') == 'bold':
                    result.append(before)
                    result.append('**')
                    result.append(text)
                    result.append('**')
                    result.append(after)
                    continue
                # Italic ?
                if properties.get('fo:font-style') == 'italic':
                    result.append(before)
                    result.append('*')
                    result.append(text)
                    result.append('*')
                    result.append(after)
                    continue
                # Unknown style, ...
                result.append(before)
                result.append(text)
                result.append(after)
            # Footnote or endnote
            elif tag == 'text:note':
                note_class = obj.get_class()
                container = {'footnote': context['footnotes'],
                             'endnote': context['endnotes']}[note_class]
                citation = obj.get_citation()
                if not citation:
                    # Would only happen with hand-made documents
                    citation = len(container)
                body = obj.get_body()
                container.append((citation, body))
                if rst_mode:
                    marker = {'footnote': u" [#]_ ",
                              'endnote': u" [*]_ "}[note_class]
                else:
                    marker = {'footnote': u"[{citation}]",
                              'endnote': u"({citation})"}[note_class]
                result.append(marker.format(citation=citation))
            # Annotations
            elif tag == 'office:annotation':
                context['annotations'].append(obj.get_body())
                if rst_mode:
                    result.append(' [#]_ ')
                else:
                    result.append('[*]')
            # Tabulation
            elif tag == 'text:tab':
                result.append(u'\t')
            # Line break
            elif tag == 'text:line-break':
                if rst_mode:
                    result.append(u"\n|")
                else:
                    result.append(u"\n")
            else:
                result.append(obj.get_formatted_text(context))
    return u''.join(result)



def odf_create_line_break():
    return odf_create_element('text:line-break')



def odf_create_undividable_space(number=1):
    """
    Arguments:

        number -- int

    Return odf_element
    """
    # FIXME what is number?
    element = odf_create_element('text:s')
    element.set_attribute('text:c', str(number))
    return element



def odf_create_paragraph(text=None, style=None):
    """Create a paragraph element of the given style containing the optional
    given text.

    Arguments:

        style -- unicode

        text -- unicode

    Return: odf_element
    """
    paragraph = odf_create_element('text:p')
    if text is not None:
        paragraph.set_text(text)
    if style is not None:
        paragraph.set_style(style)
    return paragraph



class odf_paragraph(odf_element):
    """Specialised element for paragraphs.
    """
    def get_style(self):
        return self.get_attribute('text:style-name')


    def set_style(self, name):
        return self.set_style_attribute('text:style-name', name)

<commit_msg>fix adding anything in a paragraph<commit_after># -*- coding: UTF-8 -*-
#
# Copyright (c) 2009-2010 <NAME>, Itaapy, Pierlis, Talend.
#
# Authors: <NAME> <<EMAIL>>
#          <NAME> <<EMAIL>>
#          <NAME> <<EMAIL>>
#
# This file is part of Lpod (see: http://lpod-project.org).
# Lpod is free software; you can redistribute it and/or modify it under
# the terms of either:
#
# a) the GNU General Public License as published by the Free Software
#    Foundation, either version 3 of the License, or (at your option)
#    any later version.
#    Lpod is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#    You should have received a copy of the GNU General Public License
#    along with Lpod.  If not, see <http://www.gnu.org/licenses/>.
#
# b) the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#    http://www.apache.org/licenses/LICENSE-2.0
#

# Import from the Standard Library
from re import compile, escape

# Import from lpod
from bookmark import odf_create_bookmark, odf_create_bookmark_start
from bookmark import odf_create_bookmark_end
from element import FIRST_CHILD, odf_text
from element import register_element_class, odf_element, odf_create_element
from note import odf_create_note, odf_create_annotation
from style import odf_style


def _get_formatted_text(element, context, with_text=True):
    document = context['document']
    rst_mode = context['rst_mode']

    result = []
    if with_text:
        objects = element.xpath('*|text()')
    else:
        objects = element.get_children()
    for obj in objects:
        if type(obj) is odf_text:
            result.append(obj)
        else:
            tag = obj.get_tag()
            # Good tags with text
            if tag in ('text:a', 'text:p'):
                result.append(_get_formatted_text(obj, context,
                                                 with_text=True))
            # Try to convert some styles in rst_mode
            elif tag == 'text:span':
                # XXX Move this part in span.py ??
                text = _get_formatted_text(obj, context, with_text=True)
                if not rst_mode:
                    result.append(text)
                    continue
                if not text.strip():
                    result.append(text)
                    continue
                style = obj.get_style()
                if style is None:
                    result.append(text)
                    continue
                style = document.get_style("text", style)
                properties = style.get_properties()
                if properties is None:
                    continue
                # Compute before, text and after
                before = ''
                for c in text:
                    if c.isspace():
                        before += c
                    else:
                        break
                after = ''
                for c in reversed(text):
                    if c.isspace():
                        after = c + after
                    else:
                        break
                text = text.strip()
                # Bold ?
                if properties.get('fo:font-weight') == 'bold':
                    result.append(before)
                    result.append('**')
                    result.append(text)
                    result.append('**')
                    result.append(after)
                    continue
                # Italic ?
                if properties.get('fo:font-style') == 'italic':
                    result.append(before)
                    result.append('*')
                    result.append(text)
                    result.append('*')
                    result.append(after)
                    continue
                # Unknown style, ...
                result.append(before)
                result.append(text)
                result.append(after)
            # Footnote or endnote
            elif tag == 'text:note':
                note_class = obj.get_class()
                container = {'footnote': context['footnotes'],
                             'endnote': context['endnotes']}[note_class]
                citation = obj.get_citation()
                if not citation:
                    # Would only happen with hand-made documents
                    citation = len(container)
                body = obj.get_body()
                container.append((citation, body))
                if rst_mode:
                    marker = {'footnote': u" [#]_ ",
                              'endnote': u" [*]_ "}[note_class]
                else:
                    marker = {'footnote': u"[{citation}]",
                              'endnote': u"({citation})"}[note_class]
                result.append(marker.format(citation=citation))
            # Annotations
            elif tag == 'office:annotation':
                context['annotations'].append(obj.get_body())
                if rst_mode:
                    result.append(' [#]_ ')
                else:
                    result.append('[*]')
            # Tabulation
            elif tag == 'text:tab':
                result.append(u'\t')
            # Line break
            elif tag == 'text:line-break':
                if rst_mode:
                    result.append(u"\n|")
                else:
                    result.append(u"\n")
            else:
                result.append(obj.get_formatted_text(context))
    return u''.join(result)



def odf_create_line_break():
    return odf_create_element('text:line-break')



def odf_create_undividable_space(number=1):
    """
    Arguments:

        number -- int

    Return odf_element
    """
    # FIXME what is number?
    element = odf_create_element('text:s')
    element.set_attribute('text:c', str(number))
    return element



def odf_create_paragraph(text_or_element=None, style=None):
    """Create a paragraph element of the given style containing the optional
    given text.

    Arguments:

        text -- unicode or odf_element

        style -- unicode

    Return: odf_element
    """
    paragraph = odf_create_element('text:p')
    if isinstance(text_or_element, odf_element):
        paragraph.append(text_or_element)
    else:
        paragraph.set_text(text_or_element)
    if style is not None:
        paragraph.set_style(style)
    return paragraph



class odf_paragraph(odf_element):
    """Specialised element for paragraphs.
    """
    def get_style(self):
        return self.get_attribute('text:style-name')


    def set_style(self, name):
        return self.set_style_attribute('text:style-name', name)


{'fraction_non_alphanumeric': 0.09574468085106383, 'fraction_numerical': 0.0022010271460014674, 'mean_word_length': 1.5655938264633917, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1719047', 'n_tokens_mistral': 3695, 'n_tokens_neox': 3433, 'n_words': 983}