<commit_before>"""JSON/HTML parsers for search indexing."""

import itertools
import logging
import os
import re
from urllib.parse import urlparse

import orjson as json
from django.conf import settings
from django.core.files.storage import get_storage_class
from selectolax.parser import HTMLParser

log = logging.getLogger(__name__)


class BaseParser:

    def __init__(self, version):
        self.version = version
        self.project = self.version.project
        self.storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)()

    def _parse_content(self, content):
        """Removes new line characters and strips all whitespaces."""
        content = content.strip().split('\n')

        # Convert all new lines to " "
        content = (text.strip() for text in content)
        content = ' '.join(text for text in content if text)
        return content

    def _parse_sections(self, title, body):
        """
        Parses each section into a structured dict.

        Sub-sections are nested, so they are children of the outer section,
        and sections with the same level are neighbors.
        We index the content under a section till before the next one.

        We can have pages that have content before the first title or that don't have a title,
        we index that content first under the title of the original page.
        """
        body = self._clean_body(body)

        # Index content for pages that don't start with a title.
        try:
            content = self._parse_section_content(body.child)
            if content:
                yield {
                    'id': '',
                    'title': title,
                    'content': content,
                }
        except Exception as e:
            log.info('Unable to index section: %s', str(e))

        # Index content from h1 to h6 headers.
        for head_level in range(1, 7):
            tags = body.css(f'h{head_level}')
            for tag in tags:
                try:
                    title, id = self._parse_section_title(tag)
                    content = self._parse_section_content(tag.next)
                    yield {
                        'id': id,
                        'title': title,
                        'content': content,
                    }
                except Exception as e:
                    log.info('Unable to index section: %s', str(e))

    def _clean_body(self, body):
        """
        Removes nodes with irrelevant content before parsing its sections.

        .. warning::

           This will mutate the original `body`.
        """
        # Remove all navigation nodes
        nodes_to_be_removed = body.css('[role=navigation]')
        for node in nodes_to_be_removed:
            node.decompose()

        return body

    def _is_section(self, tag):
        """
        Check if `tag` is a section (linkeable header).

        The tag is a section if:

        - It's a ``h`` tag.
        - It's a div with a ``section`` class.
        """
        is_header_tag = re.match(r'h\d$', tag.tag)
        if is_header_tag:
            return True

        is_div_section = (
            tag.tag == 'div' and
            'section' in tag.attributes.get('class', '').split()
        )
        if is_div_section:
            return True

        return False

    def _parse_section_title(self, tag):
        """
        Parses a section title tag and gets its id.

        The id (used to link to the section) is tested in the following order:

        - Get the id from the node itself.
        - Get the id from the parent node.

        Additionally:

        - Removes permalink values
        """
        nodes_to_be_removed = tag.css('a.headerlink')
        for node in nodes_to_be_removed:
            node.decompose()

        section_id = tag.attributes.get('id', '')
        if not section_id:
            parent = tag.parent
            section_id = parent.attributes.get('id', '')

        return self._parse_content(tag.text()), section_id

    def _parse_section_content(self, tag):
        """Gets the content from tag till before a new section."""
        contents = []
        next_tag = tag
        while next_tag and not self._is_section(next_tag):
            if self._is_code_section(next_tag):
                content = self._parse_code_section(next_tag)
            else:
                content = self._parse_content(next_tag.text())

            if content:
                contents.append(content)
            next_tag = next_tag.next
        return ' '.join(contents)

    def _is_code_section(self, tag):
        """
        Check if `tag` is a code section.

        Sphinx and Mkdocs codeblocks usually have a class named
        ``highlight`` or ``highlight-{language}``.
        """
        if not tag.css_first('pre'):
            return False

        for c in tag.attributes.get('class', '').split():
            if c.startswith('highlight'):
                return True
        return False

    def _parse_code_section(self, tag):
        """
        Parse a code section to fetch relevant content only.

        - Removes line numbers.

        Sphinx and Mkdocs may use a table when the code block includes line numbers.
        This table has a td tag with a ``lineos`` class.
        Other implementations put the line number within the code,
        inside span tags with the ``lineno`` class.
        """
        nodes_to_be_removed = itertools.chain(tag.css('.linenos'), tag.css('.lineno'))
        for node in nodes_to_be_removed:<commit_msg>Search: recursively parse sections (#7207)<commit_after>"""JSON/HTML parsers for search indexing."""

import itertools
import logging
import os
import re
from urllib.parse import urlparse

import orjson as json
from django.conf import settings
from django.core.files.storage import get_storage_class
from selectolax.parser import HTMLParser

log = logging.getLogger(__name__)


class BaseParser:

    def __init__(self, version):
        self.version = version
        self.project = self.version.project
        self.storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)()

    def _parse_content(self, content):
        """Removes new line characters and strips all whitespaces."""
        content = content.strip().split('\n')

        # Convert all new lines to " "
        content = (text.strip() for text in content)
        content = ' '.join(text for text in content if text)
        return content

    def _parse_sections(self, title, body):
        """
        Parses each section into a structured dict.

        Sub-sections are nested, so they are children of the outer section,
        and sections with the same level are neighbors.
        We index the content under a section till before the next one.

        We can have pages that have content before the first title or that don't have a title,
        we index that content first under the title of the original page.
        """
        body = self._clean_body(body)

        # Index content for pages that don't start with a title.
        # We check for sections till 2 levels to avoid indexing all the content
        # in this step.
        try:
            content, _ = self._parse_section_content(
                body.child,
                depth=2,
            )
            if content:
                yield {
                    'id': '',
                    'title': title,
                    'content': content,
                }
        except Exception as e:
            log.info('Unable to index section: %s', str(e))

        # Index content from h1 to h6 headers.
        for head_level in range(1, 7):
            tags = body.css(f'h{head_level}')
            for tag in tags:
                try:
                    title, id = self._parse_section_title(tag)
                    content, _ = self._parse_section_content(tag.next)
                    yield {
                        'id': id,
                        'title': title,
                        'content': content,
                    }
                except Exception as e:
                    log.info('Unable to index section: %s', str(e))

    def _clean_body(self, body):
        """
        Removes nodes with irrelevant content before parsing its sections.

        .. warning::

           This will mutate the original `body`.
        """
        # Remove all navigation nodes
        nodes_to_be_removed = body.css('[role=navigation]')
        for node in nodes_to_be_removed:
            node.decompose()

        return body

    def _is_section(self, tag):
        """
        Check if `tag` is a section (linkeable header).

        The tag is a section if:

        - It's a ``h`` tag.
        - It's a div with a ``section`` class.
        """
        is_header_tag = re.match(r'h\d$', tag.tag)
        if is_header_tag:
            return True

        is_div_section = (
            tag.tag == 'div' and
            'section' in tag.attributes.get('class', '').split()
        )
        if is_div_section:
            return True

        return False

    def _parse_section_title(self, tag):
        """
        Parses a section title tag and gets its id.

        The id (used to link to the section) is tested in the following order:

        - Get the id from the node itself.
        - Get the id from the parent node.

        Additionally:

        - Removes permalink values
        """
        nodes_to_be_removed = tag.css('a.headerlink')
        for node in nodes_to_be_removed:
            node.decompose()

        section_id = tag.attributes.get('id', '')
        if not section_id:
            parent = tag.parent
            section_id = parent.attributes.get('id', '')

        return self._parse_content(tag.text()), section_id

    def _parse_section_content(self, tag, *, depth=0):
        """
        Gets the content from tag till before a new section.

        if depth > 0, recursively check for sections in all tag's children.

        Returns a tuple with: the parsed content,
        and a boolean indicating if a section was found.
        """
        contents = []
        section_found = False

        next_tag = tag
        while next_tag:
            if section_found or self._is_section(next_tag):
                section_found = True
                break

            if self._is_code_section(next_tag):
                content = self._parse_code_section(next_tag)
            elif depth <= 0 or not next_tag.child:
                content = self._parse_content(next_tag.text())
            else:
                content, section_found = self._parse_section_content(
                    tag=next_tag.child,
                    depth=depth - 1
                )

            if content:
                contents.append(content)
            next_tag = next_tag.next

        return ' '.join(contents), section_found

    def _is_code_section(self, tag):
        """
        Check if `tag` is a code section.

        Sphinx and Mkdocs codeblocks usually have a class named
        ``highlight`` or ``highlight-{language}``.
        """
        if not tag.css_first('pre'):
            return False

        for c in tag.attributes.get('class', '').split():
            if c.startswith('highlight'):
                return True
        return False

    def _parse_code_section(self, tag):
        """
        Parse a code section to fetch relevant content only.

        - Removes line numbers.

        Sphinx and Mkdocs may use a table when the code block includes line numbers.
        This table has a td tag with a ``lineos`` class.
        Other implementations put the line number within the code,
        inside span tags with the ``lineno`` class.
        """
        nodes_to_be_removed = itertools.chain(tag.css('.linenos'), tag.css('.lineno'))
        for node in nodes_to_be_removed:
{'fraction_non_alphanumeric': 0.0848428679753981, 'fraction_numerical': 0.0015165557334232034, 'mean_word_length': 1.779208616249122, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5691513', 'n_tokens_mistral': 3200, 'n_tokens_neox': 3063, 'n_words': 1076}