<commit_before># -*- coding: utf-8 -*-
# pylint: disable=bad-continuation
""" Tools to discover and modify content.
"""
# Copyright ©  2015 <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals, print_function

import re
import difflib
try:
    import html.entities as htmlentitydefs
except ImportError:  # Python 2
    import htmlentitydefs  # pylint: disable=import-error,wrong-import-order
from xml.sax.saxutils import quoteattr  # pylint: disable=wrong-import-order

import arrow
from bunch import bunchify
from lxml.etree import fromstring, HTMLParser, XMLParser, XMLSyntaxError  # pylint: disable=no-name-in-module
from rudiments.reamed import click

from .._compat import StringIO


# Mapping of CLI content format names to Confluence API names
CLI_CONTENT_FORMATS = dict(view='view', editor='editor', storage='storage', export='export_view', anon='anonymous_export_view')

# Simple replacement rules, order is important!
TIDY_REGEX_RULES = ((_name, re.compile(_rule), _subst) for _name, _rule, _subst in [
    ("FosWiki: Remove static section numbering",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?[0-9.]+?\s*(?=<span class="tok">&nbsp;</span>)', r'\1'),
    ("FosWiki: 'tok' spans in front of headers",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s*<span class="tok">&nbsp;</span>', r'\1'),
    ("FosWiki: Section edit icons at the end of headers",
     r' *<a href="[^"]+"><ac:image [^>]+><ri:url ri:value="[^"]+/EditChapterPlugin/pencil.png" ?/>'
     r'</ac:image></a>(?=</span></h)', ''),
    ("FosWiki: 'Edit Chapter Plugin' spans (old)",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s*<span class="ecpHeading">([^<]+)</span>(?=</h.>)', r'\1\2'),
    ("FosWiki: 'Edit Chapter Plugin' spans (new)",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s*<span class="ecpHeading">([^<]+)<a class="ecpEdit".+?</a></span>(?=</h.>)', r'\1\2'),
    ("FosWiki: Residual leading whitespace in headers",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s* +', r'\1'),
    ("FosWiki: Replace TOC div with macro",
     r'(<a name="foswikiTOC" ?/>)?<div class="foswikiToc">.*?</div>', '''
          <ac:structured-macro ac:name="panel" ac:schema-version="1">
            <ac:parameter ac:name="title">Contents</ac:parameter>
            <ac:rich-text-body>
              <p>
                <ac:structured-macro ac:name="toc" ac:schema-version="1"/>
              </p>
            </ac:rich-text-body>
          </ac:structured-macro>'''),
    #("FosWiki: Replace HTML '<pre>' with 'code' macro",
    # r'<pre>', '<div><ac:structured-macro ac:name="code" ac:schema-version="1"><ac:plain-text-body>![CDATA['),
    #("FosWiki: Replace HTML '</pre>' with 'code' macro",
    # r'</pre>', ']]</ac:plain-text-body></ac:structured-macro></div>'),
])


def _apply_tidy_regex_rules(body, log=None):
    """Return tidied body after applying regex rules."""
    body = body.replace(u'\u00A0', '&nbsp;')
    for name, rule, subst in TIDY_REGEX_RULES:
        length = len(body)
        body, count = rule.subn(subst, body)
        if count and log:
            length -= len(body)
            log.info('Replaced %d matche(s) of "%s" (%d chars %s)',
                     count, name, abs(length), "added" if length < 0 else "removed")
    return body


def _make_etree(body, content_format='storage', attrs=None):
    """Create an ElementTree from a page's body."""
    attrs = (attrs or {}).copy()
    attrs.update({
        'xmlns:ac': 'http://www.atlassian.com/schema/confluence/4/ac/',
        'xmlns:ri': 'http://www.atlassian.com/schema/confluence/4/ri/',<commit_msg>tidy: new rule 'Named anchors'<commit_after># -*- coding: utf-8 -*-
# pylint: disable=bad-continuation
""" Tools to discover and modify content.
"""
# Copyright ©  2015 1&1 Group <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals, print_function

import re
import difflib
try:
    import html.entities as htmlentitydefs
except ImportError:  # Python 2
    import htmlentitydefs  # pylint: disable=import-error,wrong-import-order
from xml.sax.saxutils import quoteattr  # pylint: disable=wrong-import-order

import arrow
from bunch import bunchify
from lxml.etree import fromstring, HTMLParser, XMLParser, XMLSyntaxError  # pylint: disable=no-name-in-module
from rudiments.reamed import click

from .._compat import StringIO


# Mapping of CLI content format names to Confluence API names
CLI_CONTENT_FORMATS = dict(view='view', editor='editor', storage='storage', export='export_view', anon='anonymous_export_view')

# Simple replacement rules, order is important!
TIDY_REGEX_RULES = ((_name, re.compile(_rule), _subst) for _name, _rule, _subst in [
    ("FosWiki: Remove static section numbering",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?[0-9.]+?\s*(?=<span class="tok">&nbsp;</span>)', r'\1'),
    ("FosWiki: 'tok' spans in front of headers",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s*<span class="tok">&nbsp;</span>', r'\1'),
    ("FosWiki: Section edit icons at the end of headers",
     r' *<a href="[^"]+"><ac:image [^>]+><ri:url ri:value="[^"]+/EditChapterPlugin/pencil.png" ?/>'
     r'</ac:image></a>(?=</span></h)', ''),
    ("FosWiki: 'Edit Chapter Plugin' spans (old)",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s*<span class="ecpHeading">([^<]+)</span>(?=</h.>)', r'\1\2'),
    ("FosWiki: 'Edit Chapter Plugin' spans (new)",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s*<span class="ecpHeading">([^<]+)<a class="ecpEdit".+?</a></span>(?=</h.>)', r'\1\2'),
    ("FosWiki: Residual leading whitespace in headers",
     r'(?<=<h.>)(<a name="[^"]+?"></a>)?\s* +', r'\1'),
    ("FosWiki: Replace TOC div with macro",
     r'(<a name="foswikiTOC" ?/>)?<div class="foswikiToc">.*?</div>', '''
          <ac:structured-macro ac:name="panel" ac:schema-version="1">
            <ac:parameter ac:name="title">Contents</ac:parameter>
            <ac:rich-text-body>
              <p>
                <ac:structured-macro ac:name="toc" ac:schema-version="1"/>
              </p>
            </ac:rich-text-body>
          </ac:structured-macro>'''),
    ("FosWiki: Named anchors (#WikiWords)",
     r'(<a name=[^>]+></a><a href=")http[^#]+(#[^"]+" style="[^"]+)(" title="[^"]+"><big>[^<]+</big></a>)',
     r'\1\2; float: right;\3'),

    #("FosWiki: Replace HTML '<pre>' with 'code' macro",
    # r'<pre>', '<div><ac:structured-macro ac:name="code" ac:schema-version="1"><ac:plain-text-body>![CDATA['),
    #("FosWiki: Replace HTML '</pre>' with 'code' macro",
    # r'</pre>', ']]</ac:plain-text-body></ac:structured-macro></div>'),
])


def _apply_tidy_regex_rules(body, log=None):
    """Return tidied body after applying regex rules."""
    body = body.replace(u'\u00A0', '&nbsp;')
    for name, rule, subst in TIDY_REGEX_RULES:
        length = len(body)
        body, count = rule.subn(subst, body)
        if count and log:
            length -= len(body)
            log.info('Replaced %d matche(s) of "%s" (%d chars %s)',
                     count, name, abs(length), "added" if length < 0 else "removed")
    return body


def _make_etree(body, content_format='storage', attrs=None):
    """Create an ElementTree from a page's body."""
    attrs = (attrs or {}).copy()
    attrs.update({
        'xmlns:ac': 'http://www.atlassian.com/schema/confluence/4/ac/',
        'xmlns:ri': 'http://www.atlassian.com/schema/confluence/4/ri/',
{'fraction_non_alphanumeric': 0.2041423640042852, 'fraction_numerical': 0.007261040352339008, 'mean_word_length': 3.825962090752441, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 125, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1057946', 'n_tokens_mistral': 2989, 'n_tokens_neox': 2879, 'n_words': 726}