<commit_before># -------------------------------------------------------------
# Imports
# -------------------------------------------------------------
import os
import re
import shlex

from lxml import etree # http://infohost.nmt.edu/~shipman/soft/pylxml/web/index.html
from lxml.builder import E


# -------------------------------------------------------------
# Custom tags
# -------------------------------------------------------------

class svg(etree.ElementBase):
    pass


# -------------------------------------------------------------
# Constants
# -------------------------------------------------------------

pclass = re.compile(r'p(std|see|hang\d?|more\d?|in\d?)')
opt_pat = re.compile("""
    (?P<outside>
        [^(]*
    )
    \(
    (?P<inside>
        [^)]*
    )
    \)
    """, re.VERBOSE)

# -------------------------------------------------------------
# Functions
# -------------------------------------------------------------

def parse_improvements(root):
    """Replace certain tables into lists or code blocks"""

    convert_code(root)

    for element in root.cssselect('table.standard'):
        if detect_ul(element):
            convert_ul(element)
        elif detect_ol(element):
            convert_ol(element)
    
    #ok_paras = {'std', 'see', 'hang', 'more', 'in', 'hang2', 'more2'}
    #for p in root.cssselect('p'):
    #    cl = p.attrib.get('class', None)
    #    if cl and cl not in ok_paras:
    #        print(cl)

    return root


def detect_ul(table):
    """Detect unordered list"""
    if not len(table):
        return False
    for tr in table:
        if len(tr)!=2:
            return False
        td1 = tr[0]
        if len(td1) or td1.text.strip()!='-':
            return False
    return True

def convert_ul(table):
    """Convert table to unordered list"""
    table.tag = 'ul'
    if table.get('class'):
        del table.attrib['class']
    for tr in table:
        tr.tag = 'li'
        tr.text = tr[1].text.lstrip()
        for el in tr[1]:
            tr.append(el)
        for td in tr[:2]:
            tr.remove(td)

def detect_ol(table):
    """Detect ordered list"""
    if not len(table):
        return False
    for tr in table:
        if len(tr)!=2:
            return False
        td1 = tr[0]

        # Only keep plausible ordered lists
        if td1.text is None:
            return False
        text = td1.text.strip()
        if not text or len(text)>3:
            return False
        if text[-1] not in ('.', ')'):
            return False
        if not text[:-1].isalpha() and not text[:-1].isdigit():
            return False<commit_msg>Update smcl_parser.py<commit_after># -------------------------------------------------------------
# Imports
# -------------------------------------------------------------
import os
import re
import shlex

from lxml import etree # http://infohost.nmt.edu/~shipman/soft/pylxml/web/index.html
from lxml.builder import E


# -------------------------------------------------------------
# Custom tags
# -------------------------------------------------------------

class svg(etree.ElementBase):
    pass


# -------------------------------------------------------------
# Constants
# -------------------------------------------------------------

pclass = re.compile(r'p(std|see|hang\d?|more\d?|in\d?)')
opt_pat = re.compile("""
    (?P<outside>
        [^(]*
    )
    \(
    (?P<inside>
        [^)]*
    )
    \)
    """, re.VERBOSE)

# -------------------------------------------------------------
# Functions
# -------------------------------------------------------------

def parse_improvements(root):
    """Replace certain tables into lists or code blocks"""

    convert_code(root)

    for element in root.cssselect('table.standard'):
        if detect_ul(element):
            convert_ul(element)
        elif detect_ol(element):
            convert_ol(element)
    
    #ok_paras = {'std', 'see', 'hang', 'more', 'in', 'hang2', 'more2'}
    #for p in root.cssselect('p'):
    #    cl = p.attrib.get('class', None)
    #    if cl and cl not in ok_paras:
    #        print(cl)

    return root


def detect_ul(table):
    """Detect unordered list"""
    if not len(table):
        return False
    for tr in table:
        if len(tr)!=2:
            return False
        td1 = tr[0]
        if len(td1) or td1.text.strip()!='-':
            return False
    return True

def convert_ul(table):
    """Convert table to unordered list"""
    table.tag = 'ul'
    if table.get('class'):
        del table.attrib['class']
    for tr in table:
        tr.tag = 'li'

        has_text = any(e.text is not None for e in tr)
        #all_text = ';'.join(f'{i}={e.text}' for i,e in enumerate(tr))
        #if has_text:
        #    print(all_text)

        if has_text and tr[1].text is not None:
            tr.text = tr[1].text.lstrip()
        
        for el in tr[1]:
            tr.append(el)
        for td in tr[:2]:
            tr.remove(td)

def detect_ol(table):
    """Detect ordered list"""
    if not len(table):
        return False
    for tr in table:
        if len(tr)!=2:
            return False
        td1 = tr[0]

        # Only keep plausible ordered lists
        if td1.text is None:
            return False
        text = td1.text.strip()
        if not text or len(text)>3:
            return False
        if text[-1] not in ('.', ')'):
            return False
        if not text[:-1].isalpha() and not text[:-1].isdigit():
            return False
{'fraction_non_alphanumeric': 0.29304095686843057, 'fraction_numerical': 0.007067778180500181, 'mean_word_length': 1.1771203155818541, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2679744', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1540, 'n_words': 405}