<commit_before>
"""Various stand-alone utilities for manipulating text."""

from collections import Counter, namedtuple
import datetime
from functools import reduce, lru_cache
import itertools
import re
import string
import subprocess
from urllib.parse import urldefrag

import icu
import Levenshtein
import lxml.html
import pypandoc

from scrapers import db


def pdf2text(stream):
    """Parse a bytes object into a PDF and into text."""
    text = subprocess.run(('pdftotext', '-layout', '-', '-'),
                          input=stream, stdout=subprocess.PIPE)
    text = text.stdout.decode(encoding='utf-8')
    return text


def parse_html(url, text, clean=False):
    """Parse HTML into an `lxml` tree."""
    if clean:
        text = pypandoc.convert(text, 'html5', format='html')
    html = lxml.html.document_fromstring(text)
    # Endless loops ahoy
    html.rewrite_links(lambda s: None if urldefrag(s).url == url else s,
                       base_href=url)
    return html


class TableParser:
    """A tool for sifting through plain-text tables."""

    def __init__(self, text, max_cols=2):
        self._lines = self._split_lines(text)
        self._col_modes = self._calc_col_modes(max_cols)

    @staticmethod
    def _split_lines(text):
        r"""Split and remove leading whitespace from all lines.

        Additionally, remove blank lines.

        >>> TableParser('''
        ...
        ... Lorem ipsum   dolor sit   amet
        ...
        ... consectetur   adipiscing  totes elit
        ...
        ... ''')._lines         # doctest: +NORMALIZE_WHITESPACE
        ('Lorem ipsum   dolor sit   amet',
         'consectetur   adipiscing  totes elit')
        """
        lines = (line.lstrip() for line in text.splitlines())
        return tuple(filter(None, lines))

    @staticmethod
    def _cols_of(line):
        """The leading-edge indices of hypothetical columns within a line.

        >>> TableParser._cols_of('Lorem ipsum   dolor sit   amet')
        (14, 26)
        """
        return tuple(m.end() for m in re.finditer(r'\s{2,}', line))

    def _calc_col_modes(self, max_cols):
        r"""The `self._max_cols` most common column indices in the table.

        >>> TableParser('''\
        ... Lorem ipsum   dolor sit   amet
        ... consectetur   adipiscing  totes elit
        ... ''', 3)._col_modes
        (14, 26)<commit_msg>Strip from the right 'cause we don't wanna be shifting columns<commit_after>
"""Various stand-alone utilities for manipulating text."""

from collections import Counter, namedtuple
import datetime
from functools import reduce, lru_cache
import itertools
import re
import string
import subprocess
from urllib.parse import urldefrag

import icu
import Levenshtein
import lxml.html
import pypandoc

from scrapers import db


def pdf2text(stream):
    """Parse a bytes object into a PDF and into text."""
    text = subprocess.run(('pdftotext', '-layout', '-', '-'),
                          input=stream, stdout=subprocess.PIPE)
    text = text.stdout.decode(encoding='utf-8')
    return text


def parse_html(url, text, clean=False):
    """Parse HTML into an `lxml` tree."""
    if clean:
        text = pypandoc.convert(text, 'html5', format='html')
    html = lxml.html.document_fromstring(text)
    # Endless loops ahoy
    html.rewrite_links(lambda s: None if urldefrag(s).url == url else s,
                       base_href=url)
    return html


class TableParser:
    """A tool for sifting through plain-text tables."""

    def __init__(self, text, max_cols=2):
        self._lines = self._split_lines(text)
        self._col_modes = self._calc_col_modes(max_cols)

    @staticmethod
    def _split_lines(text):
        r"""Split and remove leading whitespace from all lines.

        Additionally, remove blank lines.

        >>> TableParser('''
        ...
        ... Lorem ipsum   dolor sit   amet
        ...
        ... consectetur   adipiscing  totes elit
        ...
        ... ''')._lines         # doctest: +NORMALIZE_WHITESPACE
        ('Lorem ipsum   dolor sit   amet',
         'consectetur   adipiscing  totes elit')
        """
        lines = (line.rstrip() for line in text.splitlines())
        return tuple(filter(None, lines))

    @staticmethod
    def _cols_of(line):
        """The leading-edge indices of hypothetical columns within a line.

        >>> TableParser._cols_of('Lorem ipsum   dolor sit   amet')
        (14, 26)
        """
        return tuple(m.end() for m in re.finditer(r'\s{2,}', line))

    def _calc_col_modes(self, max_cols):
        r"""The `self._max_cols` most common column indices in the table.

        >>> TableParser('''\
        ... Lorem ipsum   dolor sit   amet
        ... consectetur   adipiscing  totes elit
        ... ''', 3)._col_modes
        (14, 26)
{'fraction_non_alphanumeric': 0.1128931472609873, 'fraction_numerical': 0.005832118308685691, 'mean_word_length': 2.427551748750892, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1840361', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1450, 'n_words': 430}