<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""An extensible ASCII table reader and writer.

latex.py:
  Classes to read and write LaTeX tables

:Copyright: Smithsonian Astrophysical Observatory (2011)
:Author: <NAME> (<EMAIL>)
"""

from __future__ import absolute_import, division, print_function

import re

from ...extern import six
from ...extern.six.moves import zip
from . import core

latexdicts = {'AA':  {'tabletype': 'table',
                      'header_start': r'\hline \hline', 'header_end': r'\hline',
                      'data_end': r'\hline'},
              'doublelines': {'tabletype': 'table',
                              'header_start': r'\hline \hline', 'header_end': r'\hline\hline',
                              'data_end': r'\hline\hline'},
              'template': {'tabletype': 'tabletype', 'caption': 'caption',
                           'tablealign': 'tablealign',
                           'col_align': 'col_align', 'preamble': 'preamble',
                           'header_start': 'header_start',
                           'header_end': 'header_end', 'data_start': 'data_start',
                           'data_end': 'data_end', 'tablefoot': 'tablefoot',
                           'units': {'col1': 'unit of col1', 'col2': 'unit of col2'}}
              }


RE_COMMENT = re.compile(r'(?<!\\)%')  # % character but not \%

def add_dictval_to_list(adict, key, alist):
    '''
    Add a value from a dictionary to a list

    Parameters
    ----------
    adict : dictionary
    key : hashable
    alist : list
        List where value should be added
    '''
    if key in adict:
        if isinstance(adict[key], six.string_types):
            alist.append(adict[key])
        else:
            alist.extend(adict[key])


def find_latex_line(lines, latex):
    '''
    Find the first line which matches a patters

    Parameters
    ----------
    lines : list
        List of strings
    latex : str
        Search pattern

    Returns
    -------
    line_num : int, None
        Line number. Returns None, if no match was found

    '''
    re_string = re.compile(latex.replace('\\', '\\\\'))
    for i, line in enumerate(lines):
        if re_string.match(line):
            return i
    else:
        return None


class LatexSplitter(core.BaseSplitter):
    '''Split LaTeX table date. Default delimiter is `&`.
    '''
    delimiter = '&'

    def __call__(self, lines):
        last_line = RE_COMMENT.split(lines[-1])[0].strip()
        if not last_line.endswith(r'\\'):
            print(last_line)
            lines[-1] = last_line + r'\\'

        return super(LatexSplitter, self).__call__(lines)

    def process_line(self, line):
        """Remove whitespace at the beginning or end of line. Also remove
        \\ at end of line"""
        line = RE_COMMENT.split(line)[0]
        line = line.strip()
        if line.endswith(r'\\'):
            line = line.rstrip(r'\\')
        else:
            raise core.InconsistentTableError(r'Lines in LaTeX table have to end with \\')
        return line

    def process_val(self, val):
        """Remove whitespace and {} at the beginning or end of value."""
        val = val.strip()<commit_msg>Remove debugging "print" introduced in #5427<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""An extensible ASCII table reader and writer.

latex.py:
  Classes to read and write LaTeX tables

:Copyright: Smithsonian Astrophysical Observatory (2011)
:Author: <NAME> (<EMAIL>)
"""

from __future__ import absolute_import, division, print_function

import re

from ...extern import six
from ...extern.six.moves import zip
from . import core

latexdicts = {'AA':  {'tabletype': 'table',
                      'header_start': r'\hline \hline', 'header_end': r'\hline',
                      'data_end': r'\hline'},
              'doublelines': {'tabletype': 'table',
                              'header_start': r'\hline \hline', 'header_end': r'\hline\hline',
                              'data_end': r'\hline\hline'},
              'template': {'tabletype': 'tabletype', 'caption': 'caption',
                           'tablealign': 'tablealign',
                           'col_align': 'col_align', 'preamble': 'preamble',
                           'header_start': 'header_start',
                           'header_end': 'header_end', 'data_start': 'data_start',
                           'data_end': 'data_end', 'tablefoot': 'tablefoot',
                           'units': {'col1': 'unit of col1', 'col2': 'unit of col2'}}
              }


RE_COMMENT = re.compile(r'(?<!\\)%')  # % character but not \%

def add_dictval_to_list(adict, key, alist):
    '''
    Add a value from a dictionary to a list

    Parameters
    ----------
    adict : dictionary
    key : hashable
    alist : list
        List where value should be added
    '''
    if key in adict:
        if isinstance(adict[key], six.string_types):
            alist.append(adict[key])
        else:
            alist.extend(adict[key])


def find_latex_line(lines, latex):
    '''
    Find the first line which matches a patters

    Parameters
    ----------
    lines : list
        List of strings
    latex : str
        Search pattern

    Returns
    -------
    line_num : int, None
        Line number. Returns None, if no match was found

    '''
    re_string = re.compile(latex.replace('\\', '\\\\'))
    for i, line in enumerate(lines):
        if re_string.match(line):
            return i
    else:
        return None


class LatexSplitter(core.BaseSplitter):
    '''Split LaTeX table date. Default delimiter is `&`.
    '''
    delimiter = '&'

    def __call__(self, lines):
        last_line = RE_COMMENT.split(lines[-1])[0].strip()
        if not last_line.endswith(r'\\'):
            lines[-1] = last_line + r'\\'

        return super(LatexSplitter, self).__call__(lines)

    def process_line(self, line):
        """Remove whitespace at the beginning or end of line. Also remove
        \\ at end of line"""
        line = RE_COMMENT.split(line)[0]
        line = line.strip()
        if line.endswith(r'\\'):
            line = line.rstrip(r'\\')
        else:
            raise core.InconsistentTableError(r'Lines in LaTeX table have to end with \\')
        return line

    def process_val(self, val):
        """Remove whitespace and {} at the beginning or end of value."""
        val = val.strip()
{'fraction_non_alphanumeric': 0.13509749303621169, 'fraction_numerical': 0.004642525533890436, 'mean_word_length': 1.9660394676457091, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3710820', 'n_tokens_mistral': 1939, 'n_tokens_neox': 1859, 'n_words': 543}