<commit_before># -*- coding: utf-8 -*-

"""
platform_section
----------------------------------

docdown.platform_section Markdown extension module
"""

from __future__ import absolute_import, print_function, unicode_literals

import re

from markdown.extensions import Extension
from markdown.preprocessors import Preprocessor


class PlatformSectionPreprocessor(Preprocessor):

    BLOCK_RE = re.compile(r'''
^@!\[(?P<sections>[\w, ]+)\]\W*\n
(?P<content>.*?)(?<=\n)
!@\W*$''', re.MULTILINE | re.DOTALL | re.VERBOSE)

    INLINE_RE = re.compile(r'''@!\[(?P<sections>[\w, ]+)\](?P<content>.*?)!@''', re.DOTALL | re.VERBOSE)
    STARTSWITH_WHITESPACE_RE = re.compile(r'^\W+(@!\[|\n)')

    def __init__(self, platform_section, **kwargs):
        self.platform_section = platform_section.lower().strip()
        super(PlatformSectionPreprocessor, self).__init__(**kwargs)

    def run(self, lines):
        text = "\n".join(lines)
        text = self.process_inline(text)
        text = self.process_block(text)
        return text.split("\n")

    def split_sections(self, sections_group):
        return [section.lower().strip() for section in sections_group.split(',')]

    def process_inline(self, text):
        while 1:
            m = self.INLINE_RE.search(text)
            if m:
                sections = self.split_sections(m.group('sections'))

                if self.platform_section in sections:
                    content = m.group('content')
                    text = '%s%s%s' % (text[:m.start()], content, text[m.end():])
                else:
                    text = '%s%s' % (text[:m.start()], text[m.end():].lstrip())
            else:
                break
        return text

    def process_block(self, text):
        while 1:
            m = self.BLOCK_RE.search(text)
            if m:
                sections = self.split_sections(m.group('sections'))
                start = text[:m.start()]
                end = text[m.end():]
                if self.STARTSWITH_WHITESPACE_RE.match(end):
                    end = end.lstrip()

                if self.platform_section in sections:
                    content = m.group('content')
                    text = '%s%s%s' % (start, content, end)<commit_msg>remove unused block re<commit_after># -*- coding: utf-8 -*-

"""
platform_section
----------------------------------

docdown.platform_section Markdown extension module
"""

from __future__ import absolute_import, print_function, unicode_literals

import re

from markdown.extensions import Extension
from markdown.preprocessors import Preprocessor


class PlatformSectionPreprocessor(Preprocessor):

    PLATFORM_SECTION_RE = re.compile(r'''@!\[(?P<sections>[\w, ]+)\](?P<content>.*?)!@''', re.DOTALL | re.VERBOSE)

    STARTSWITH_WHITESPACE_RE = re.compile(r'^\W+(@!\[|\n)')

    def __init__(self, platform_section, **kwargs):
        self.platform_section = platform_section.lower().strip()
        super(PlatformSectionPreprocessor, self).__init__(**kwargs)

    def run(self, lines):
        text = "\n".join(lines)
        text = self.process_platform_sections(text)
        return text.split("\n")

    def split_sections(self, sections_group):
        return [section.lower().strip() for section in sections_group.split(',')]

    def process_platform_sections(self, text):
        while 1:
            m = self.PLATFORM_SECTION_RE.search(text)
            if m:
                sections = self.split_sections(m.group('sections'))

                start = text[:m.start()]
                end = text[m.end():]
                if self.STARTSWITH_WHITESPACE_RE.match(end):
                    end = end.lstrip()

                if self.platform_section in sections:
                    content = m.group('content')
                    text = '%s%s%s' % (start, content, end)
{'fraction_non_alphanumeric': 0.16173776498298875, 'fraction_numerical': 0.0013085579691180318, 'mean_word_length': 2.300518134715026, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3072488', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1119, 'n_words': 222}