<commit_before># Copyright (C) 2014 <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
Main processing module.
"""

import re
from json import dumps
from random import random
from hashlib import sha1

from pygments import lexers, highlight, formatters
from markdown import markdown
from docutils.core import publish_parts
from bs4 import BeautifulSoup, Tag


default_tpl = """\
<!DOCTYPE html>
<html>
<head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>{title}</title>

    <style type="text/css">
    body {{
        margin: 0;
        padding: 10px;
        background-color: lightsteelblue;
    }}
    #wrapper {{
        border: 1px solid lightgray;
        width: 80%;
        margin: auto;
        padding: 10px 20px;
        background-color: white;
    }}
    #two-columns {{
    }}
    #left-col {{
        float: left;
        width: 50%;
    }}
    #right-col {{
        float: left;
        width: 49%;
    }}
    .no-float {{
        clear: both;
    }}
    .annotations {{
        padding: 5px 20px 10px 20px;
        font-family: DejaVu Sans, Verdana, sans-serif;
    }}
    table.highlighttable {{
        width: 100%;
        padding: 0px 10px;
    }}
    </style>

    <style type="text/css">
    {style}
    </style>

    <script src="http://code.jquery.com/jquery-2.1.0.min.js"></script>
    <script type="text/javascript">
    {script}
    </script>
</head>
<body>
<div id="wrapper">
    <div id="two-columns">
        <div id="left-col">
            <div class="annotations">
                {annotations}
            </div>
        </div>
        <div id="right-col">
            {code}
        </div>
    </div>
    <div class="no-float"></div>
</div>
</body>
</html>
"""


annotation_tpl = """\
<div class="annotation">
    <span style="display: none;" class="data">{json}</span>
    {body}
</div>
"""


interact_script = """\
$(window).load(function () {

    // Hide annotations non title elements
    $('div.annotation_body').children(':not(.annotation_title)').hide();

    var speed = 300;

    function show_annotation() {
        $(this).toggleClass('hover');
        $(this).children('*:not(.annotation_title)').slideToggle(speed);

        var meta = jQuery.parseJSON($(this).siblings('.data').text());
        if (meta.args != null) {
            for (var i = 0; i < meta.args.length; i++) {
                $('#' + meta.prefix + 'line-' + meta.args[i]).toggleClass(
                    'hll hll-line'
                );
            }
        }
    }

    $('div.annotation_body').hover(show_annotation, show_annotation);
});
"""


class Processor(object):

    """
    Regular expression used to find annotations.
    """
    ann_regex = \
        r'^<\[annotation\]> *?(?P<args>[0-9]+(\[[0-9]+, *[0-9]+\])?)? *?$'
    ann_re = re.compile(ann_regex)

    def _parse_annotations(self, annotations, prefix):
        """
        Parse annotations from a string.

        This method allows to split a large string containing annotations and
        returns a list of tuples with (dictionary, annotation).

        :param str annotations: String with annotations.
        :param str prefix: Prefix to be used for this annotated code.
        """

        current = None
        buff = []
        parsed_anns = []

        for line in annotations.splitlines():
            m = Processor.ann_re.match(line)
            if not m:
                buff.append(line)
                continue

            if current is not None:
                parsed_anns.append(
                    (current, '\n'.join(buff))
                )
            current = {
                'args' : None,
                'prefix' : prefix
            }
            args = m.groupdict()['args']
            if args is not None:
                current['args'] = args.strip().split(' ')
            buff = []

        if buff:
            parsed_anns.append(
                (current, '\n'.join(buff))
            )

        return parsed_anns

    def _render_markdown(self, body, **kwargs):
        """
        Render given Markdown formatted body to HTML.

        :param str body: String with Markdown.
        """

        kwargs['output_format'] = 'html4'  # Same as Pygments
        return markdown(body, **kwargs)

    def _render_rest(self, body, **kwargs):
        """
        Render given reStructuredText formatted body to HTML.

        :param str body: String with reStructuredText.
        """

        overrides = {
            'doctitle_xform': False,
            'initial_header_level': 1
        }
        overrides.update(kwargs)
        parts = publish_parts(
            source=body,
            writer_name='html',
            settings_overrides=overrides
        )
        return parts['body']

    def _render(self, parsed_anns, ann_format, renderer_opts):
        """
        Render to the specified format the given parsed annotations.

        :param list parsed_anns: A list of parsed annotations in the format
         given by :meth:`Processor._parse_annotations`.
        :param str ann_format: Format to render the annotations. Supported
         formats are 'rest' and 'markdown'.
        :param dict renderer_opts: Options to be passed to the renderer.
        """

        # Get renderer
        renderers = {
            'markdown' : self._render_markdown,
            'rest'     : self._render_rest,
        }
        renderer = renderers[ann_format]

        def add_class(elem, html_class):
            """
            Helper to safely add a class to a Tag object.
            """
            if not isinstance(elem, Tag):
                return False
            if 'class' in elem:
                elem.attrs['class'] += [html_class]
            else:
                elem.attrs['class'] = html_class
            return True

        # Render annotations
        rendered_anns = []
        for meta, ann_body in parsed_anns:

            html = renderer(ann_body, **renderer_opts).strip()

            # Wrap rendered annotation
            bs = BeautifulSoup(html)
            elements = bs.body.contents
            if len(elements) == 1 and elements[0].name == 'div':
                # Already wrapped
                wrapper = elements[0]
            else:
                # Wrap into a div
                wrapper = bs.new_tag('div')
                for elem in elements:
                    wrapper.append(elem)
            add_class(wrapper, 'annotation_body')

            for child in wrapper.children:
                if add_class(child, 'annotation_title'):
                    break
            body = str(wrapper)

            rendered_anns.append(
                annotation_tpl.format(json=dumps(meta), body=body)
            )

        return rendered_anns

    def _generate_prefix(self, lenght=10):
        """
        Generates a random hash to be used as prefix.

        :param int lenght: Size to cut the hash.
        """

        the_hash = sha1()
        the_hash.update(str(random()))
        return the_hash.hexdigest()[:lenght]

    def process(
            self, code, annotations,
            codefn=None, ann_format='rest',
            prefix=None, codestyle='monokai',
            renderer_opts=None):
        """
        Main processing function.

        :param str code: Code to be highlighted.
        :param str annotations: Text with annotations.
        :param str codefn: Optional "CodeFileName" that can be used to better
         detect the programming language in code.
        :param str ann_format: Format of the annotations. Supported formats are
         ``'rest'`` and ``'markdown'``.
        :param str prefix: Prefix to be used to identify this block
         (code - annotations pair). The prefix allows multiples blocks to be
         included in the same web page without interfering with each other. If
         ``None`` is given, a random prefix will be generated.
        :param str codestyle: Pygments style to be used for syntax highlight.
         See http://pygments.org/docs/styles/
        :param dict renderer_opts: Dictionary with keyword options to be passed
         to the renderer. For Markdown, this dictionary is passed to the
         ``markdown.markdown`` function as ``**kwargs``. For reStructuredText
         this dictionary is passed to the ``settings_overrides`` argument of
         the ``docutils.code.publish_parts`` function.
        """

        if renderer_opts is None:
            renderer_opts = {}
        if prefix is None:
            prefix = self._generate_prefix()

        # Guess programming language
        # Warning: might raise pygments.util.ClassNotFound
        if codefn is None:
            lexer = lexers.guess_lexer(code)
        else:
            lexer = lexers.guess_lexer_for_filename(codefn, code)

        # Parse annotations
        parsed_anns = self._parse_annotations(
            annotations, prefix
        )

        # Render annotations
        rendered_anns = self._render(
            parsed_anns, ann_format, renderer_opts
        )

        # Highlight code
        options = {
            'style'    : codestyle,
            'linenos'  : 'table',
            'linespans': prefix + 'line',
        }
        formatter = formatters.HtmlFormatter(**options)
        highlighted = highlight(code, lexer, formatter)
        style = formatter.get_style_defs('table.highlighttable') + \
            '\ntable.highlighttable .hll-line { display: block; }'

        return {
            'style'       : style,
            'script'      : interact_script,
            'annotations' : rendered_anns,
            'code'        : highlighted,
        }

    def process_files(self, codefn, annfn, **kwargs):
        """
        Process and interpret given code file name and annotations file name.

        :param str codefn: Path to the code file.
        :param str annfn: Path to the annotations file.
        :param dict kwargs: Except for ``codefn`` (with is automatically set),
         this method supports all the other arguments
         :meth:`Processor.process`` supports.
        """

        # Warning: might raise IO exceptions
        with open(codefn, 'r') as cf:
            code = cf.read()
        with open(annfn, 'r') as af:
            annotations = af.read()
        return self.process(code, annotations, codefn=codefn, **kwargs)

    def create_document(
            self, codefn, annfn,
            title='', tpl=None, out_file=None, **kwargs):
        """

        The full content of the document is always returned.

        :param str codefn: Path to the code file.
        :param str annfn: Path to the annotations file.
        :param str title: Title of the document.
        :param str tpl: Python template string to be used a template for the
         document. See ``codeco.processor.default_tpl`` for an example. If
         ``None`` is given, the ``default_tpl`` will be used.
        :param str out_file: Optional path for the output file. If given, the
         file will be created or overriden with the content of the document.
        :param dict kwargs: Except for ``codefn`` (with is automatically set),
         this method supports all the other arguments
         :meth:`Processor.process`` supports.
        """

        processed = self.process_files(codefn, annfn, **kwargs)

        # Add title and join annotations
        processed['title'] = title
        processed['annotations'] = \
            '\n'.join(processed['annotations'])

        if tpl is None:
            tpl = default_tpl
        document = tpl.format(**processed)

        # Warning: might raise IO exceptions
        if out_file is not None:
            with open(out_file, 'w') as of:
                of.write(document)

        return document
<commit_msg>Fixing, again?, the regular expression.<commit_after># Copyright (C) 2014 <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
Main processing module.
"""

import re
from json import dumps
from random import random
from hashlib import sha1

from pygments import lexers, highlight, formatters
from markdown import markdown
from docutils.core import publish_parts
from bs4 import BeautifulSoup, Tag


default_tpl = """\
<!DOCTYPE html>
<html>
<head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>{title}</title>

    <style type="text/css">
    body {{
        margin: 0;
        padding: 10px;
        background-color: lightsteelblue;
    }}
    #wrapper {{
        border: 1px solid lightgray;
        width: 80%;
        margin: auto;
        padding: 10px 20px;
        background-color: white;
    }}
    #two-columns {{
    }}
    #left-col {{
        float: left;
        width: 50%;
    }}
    #right-col {{
        float: left;
        width: 49%;
    }}
    .no-float {{
        clear: both;
    }}
    .annotations {{
        padding: 5px 20px 10px 20px;
        font-family: DejaVu Sans, Verdana, sans-serif;
    }}
    table.highlighttable {{
        width: 100%;
        padding: 0px 10px;
    }}
    </style>

    <style type="text/css">
    {style}
    </style>

    <script src="http://code.jquery.com/jquery-2.1.0.min.js"></script>
    <script type="text/javascript">
    {script}
    </script>
</head>
<body>
<div id="wrapper">
    <div id="two-columns">
        <div id="left-col">
            <div class="annotations">
                {annotations}
            </div>
        </div>
        <div id="right-col">
            {code}
        </div>
    </div>
    <div class="no-float"></div>
</div>
</body>
</html>
"""


annotation_tpl = """\
<div class="annotation">
    <span style="display: none;" class="data">{json}</span>
    {body}
</div>
"""


interact_script = """\
$(window).load(function () {

    // Hide annotations non title elements
    $('div.annotation_body').children(':not(.annotation_title)').hide();

    var speed = 300;

    function show_annotation() {
        $(this).toggleClass('hover');
        $(this).children('*:not(.annotation_title)').slideToggle(speed);

        var meta = jQuery.parseJSON($(this).siblings('.data').text());
        if (meta.args != null) {
            for (var i = 0; i < meta.args.length; i++) {
                $('#' + meta.prefix + 'line-' + meta.args[i]).toggleClass(
                    'hll hll-line'
                );
            }
        }
    }

    $('div.annotation_body').hover(show_annotation, show_annotation);
});
"""


class Processor(object):

    """
    Regular expression used to find annotations.
    """
    ann_regex = \
        r'^<\[annotation\]>(?P<args>[0-9, \[\]]+)? *?$'
    ann_re = re.compile(ann_regex)

    def _parse_annotations(self, annotations, prefix):
        """
        Parse annotations from a string.

        This method allows to split a large string containing annotations and
        returns a list of tuples with (dictionary, annotation).

        :param str annotations: String with annotations.
        :param str prefix: Prefix to be used for this annotated code.
        """

        current = None
        buff = []
        parsed_anns = []

        for line in annotations.splitlines():
            m = Processor.ann_re.match(line)
            if not m:
                buff.append(line)
                continue

            if current is not None:
                parsed_anns.append(
                    (current, '\n'.join(buff))
                )
            current = {
                'args' : None,
                'prefix' : prefix
            }
            args = m.groupdict()['args']
            if args is not None:
                current['args'] = args.strip().split(' ')
            buff = []

        if buff:
            parsed_anns.append(
                (current, '\n'.join(buff))
            )

        return parsed_anns

    def _render_markdown(self, body, **kwargs):
        """
        Render given Markdown formatted body to HTML.

        :param str body: String with Markdown.
        """

        kwargs['output_format'] = 'html4'  # Same as Pygments
        return markdown(body, **kwargs)

    def _render_rest(self, body, **kwargs):
        """
        Render given reStructuredText formatted body to HTML.

        :param str body: String with reStructuredText.
        """

        overrides = {
            'doctitle_xform': False,
            'initial_header_level': 1
        }
        overrides.update(kwargs)
        parts = publish_parts(
            source=body,
            writer_name='html',
            settings_overrides=overrides
        )
        return parts['body']

    def _render(self, parsed_anns, ann_format, renderer_opts):
        """
        Render to the specified format the given parsed annotations.

        :param list parsed_anns: A list of parsed annotations in the format
         given by :meth:`Processor._parse_annotations`.
        :param str ann_format: Format to render the annotations. Supported
         formats are 'rest' and 'markdown'.
        :param dict renderer_opts: Options to be passed to the renderer.
        """

        # Get renderer
        renderers = {
            'markdown' : self._render_markdown,
            'rest'     : self._render_rest,
        }
        renderer = renderers[ann_format]

        def add_class(elem, html_class):
            """
            Helper to safely add a class to a Tag object.
            """
            if not isinstance(elem, Tag):
                return False
            if 'class' in elem:
                elem.attrs['class'] += [html_class]
            else:
                elem.attrs['class'] = html_class
            return True

        # Render annotations
        rendered_anns = []
        for meta, ann_body in parsed_anns:

            html = renderer(ann_body, **renderer_opts).strip()

            # Wrap rendered annotation
            bs = BeautifulSoup(html)
            elements = bs.body.contents
            if len(elements) == 1 and elements[0].name == 'div':
                # Already wrapped
                wrapper = elements[0]
            else:
                # Wrap into a div
                wrapper = bs.new_tag('div')
                for elem in elements:
                    wrapper.append(elem)
            add_class(wrapper, 'annotation_body')

            for child in wrapper.children:
                if add_class(child, 'annotation_title'):
                    break
            body = str(wrapper)

            rendered_anns.append(
                annotation_tpl.format(json=dumps(meta), body=body)
            )

        return rendered_anns

    def _generate_prefix(self, lenght=10):
        """
        Generates a random hash to be used as prefix.

        :param int lenght: Size to cut the hash.
        """

        the_hash = sha1()
        the_hash.update(str(random()))
        return the_hash.hexdigest()[:lenght]

    def process(
            self, code, annotations,
            codefn=None, ann_format='rest',
            prefix=None, codestyle='monokai',
            renderer_opts=None):
        """
        Main processing function.

        :param str code: Code to be highlighted.
        :param str annotations: Text with annotations.
        :param str codefn: Optional "CodeFileName" that can be used to better
         detect the programming language in code.
        :param str ann_format: Format of the annotations. Supported formats are
         ``'rest'`` and ``'markdown'``.
        :param str prefix: Prefix to be used to identify this block
         (code - annotations pair). The prefix allows multiples blocks to be
         included in the same web page without interfering with each other. If
         ``None`` is given, a random prefix will be generated.
        :param str codestyle: Pygments style to be used for syntax highlight.
         See http://pygments.org/docs/styles/
        :param dict renderer_opts: Dictionary with keyword options to be passed
         to the renderer. For Markdown, this dictionary is passed to the
         ``markdown.markdown`` function as ``**kwargs``. For reStructuredText
         this dictionary is passed to the ``settings_overrides`` argument of
         the ``docutils.code.publish_parts`` function.
        """

        if renderer_opts is None:
            renderer_opts = {}
        if prefix is None:
            prefix = self._generate_prefix()

        # Guess programming language
        # Warning: might raise pygments.util.ClassNotFound
        if codefn is None:
            lexer = lexers.guess_lexer(code)
        else:
            lexer = lexers.guess_lexer_for_filename(codefn, code)

        # Parse annotations
        parsed_anns = self._parse_annotations(
            annotations, prefix
        )

        # Render annotations
        rendered_anns = self._render(
            parsed_anns, ann_format, renderer_opts
        )

        # Highlight code
        options = {
            'style'    : codestyle,
            'linenos'  : 'table',
            'linespans': prefix + 'line',
        }
        formatter = formatters.HtmlFormatter(**options)
        highlighted = highlight(code, lexer, formatter)
        style = formatter.get_style_defs('table.highlighttable') + \
            '\ntable.highlighttable .hll-line { display: block; }'

        return {
            'style'       : style,
            'script'      : interact_script,
            'annotations' : rendered_anns,
            'code'        : highlighted,
        }

    def process_files(self, codefn, annfn, **kwargs):
        """
        Process and interpret given code file name and annotations file name.

        :param str codefn: Path to the code file.
        :param str annfn: Path to the annotations file.
        :param dict kwargs: Except for ``codefn`` (with is automatically set),
         this method supports all the other arguments
         :meth:`Processor.process`` supports.
        """

        # Warning: might raise IO exceptions
        with open(codefn, 'r') as cf:
            code = cf.read()
        with open(annfn, 'r') as af:
            annotations = af.read()
        return self.process(code, annotations, codefn=codefn, **kwargs)

    def create_document(
            self, codefn, annfn,
            title='', tpl=None, out_file=None, **kwargs):
        """

        The full content of the document is always returned.

        :param str codefn: Path to the code file.
        :param str annfn: Path to the annotations file.
        :param str title: Title of the document.
        :param str tpl: Python template string to be used a template for the
         document. See ``codeco.processor.default_tpl`` for an example. If
         ``None`` is given, the ``default_tpl`` will be used.
        :param str out_file: Optional path for the output file. If given, the
         file will be created or overriden with the content of the document.
        :param dict kwargs: Except for ``codefn`` (with is automatically set),
         this method supports all the other arguments
         :meth:`Processor.process`` supports.
        """

        processed = self.process_files(codefn, annfn, **kwargs)

        # Add title and join annotations
        processed['title'] = title
        processed['annotations'] = \
            '\n'.join(processed['annotations'])

        if tpl is None:
            tpl = default_tpl
        document = tpl.format(**processed)

        # Warning: might raise IO exceptions
        if out_file is not None:
            with open(out_file, 'w') as of:
                of.write(document)

        return document

{'fraction_non_alphanumeric': 0.10547716829992274, 'fraction_numerical': 0.004635465376326597, 'mean_word_length': 1.923332937121122, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '320283', 'n_tokens_mistral': 6938, 'n_tokens_neox': 6584, 'n_words': 2086}