<commit_before>#!/usr/bin/env python

"""
Path-like row selector for pandas DataFrames with hierarchical MultiIndexes.
"""

import re
import numpy as np

import pandas as pd
import ply.lex as lex
import ply.yacc as yacc

class PathLikeSelector(object):
    """
    Class for selecting rows of a pandas DataFrame using path-like selectors.

    Select rows from a pandas DataFrame using path-like selectors.
    Assumes that the DataFrame instance has a MultiIndex where each level
    corresponds to a level of the selector; a level may either be a denoted by a
    string label (e.g., 'foo') or a numerical index (e.g., 0, 1, 2).
    Examples of valid selectors include

    /foo/bar
    /foo/[qux,bar]
    /foo/bar[0]
    /foo/bar[0,1]
    /foo/bar[0:5]
    /foo/*/baz
    /foo/*/baz[5]
    /foo/bar+/baz/qux

    The class can also be used to create new MultiIndex instances from selectors
    that contain no wildcards.

    Notes
    -----
    Numerical indices are assumed to be zero-based. Ranges do not include the
    end element (i.e., like numpy, not like pandas).
    """

    tokens = ('ASTERISK', 'INTEGER', 'INTEGER_SET',
              'INTERVAL', 'STRING', 'STRING_SET', 'PLUS')

    def __init__(self):
        self._setup()

    def _parse_interval_str(self, s):
        """
        Convert string representation of interval to tuple containing numerical
        start and stop values.
        """

        start, stop = s.split(':')
        if start == '':
            start = 0
        else:
            start = int(start)
        if stop == '':
            stop = np.inf
        else:
            stop = int(stop)
        return (start, stop)

    def t_PLUS(self, t):
        r'\+'
        return t

    def t_ASTERISK(self, t):
        r'/\*'
        t.value = t.value.strip('/')
        return t

    def t_INTEGER(self, t):
        r'/?\[\d+\]'
        t.value = int(re.search('\[(\d+)\]', t.value.strip('/')).group(1))
        return t

    def t_INTEGER_SET(self, t):
        r'/?\[(?:\d+,?)+\]'
        t.value = map(int, t.value.strip('[]').split(','))
        return t

    def t_INTERVAL(self, t):
        r'\[\d*\:\d*\]'
        t.value = self._parse_interval_str(re.search('\[(.+)\]', t.value).group(1))
        return t

    def t_STRING(self, t):
        r'/[^*/\[\]:\d][^+*/\[\]:]*'
        t.value = t.value.strip('/')
        return t

    def t_STRING_SET(self, t):
        r'/\[(?:[^+*/\[\]:\d][^+*/\[\]:]*,?)+\]'
        t.value = t.value.strip('/[]').split(',')
        return t

    def t_error(self, t):
        print 'Illegal character "%s"' % t.value[0]
        raise ValueError('Cannot tokenize selector')

    def p_list_plus(self, p):
        'list : list PLUS selector'
        p[0] = p[1]+[p[3]]

    def p_list_selector(self, p):
        'list : selector'
        p[0] = [p[1]]

    def p_selector_plus(self, p):
        'selector : selector token'
        p[0] = p[1]
        p[0].append(p[2])

    def p_selector_token(self, p):
        'selector : token'
        p[0] = [p[1]]

    def p_token(self, p):
        '''token : ASTERISK
               | INTEGER
               | INTEGER_SET
               | INTERVAL
               | STRING
               | STRING_SET'''
        p[0] = p[1]

    def p_error(self, p):
        print 'Syntax error'
        raise ValueError('Cannot parse selector')

    def _setup(self, **kwargs):
        """
        Build lexer and parser.
        """

        self.lexer = lex.lex(module=self, **kwargs)
        self.parser = yacc.yacc(module=self, **kwargs)

    def tokenize(self, selector):
        """
        Tokenize a selector string.

        Parameters
        ----------
        selector : str
            Row selector.

        Returns
        -------
        token_list : list
            List of tokens extracted by ply.
        """

        self.lexer.input(selector)<commit_msg>Suppress generation of debug file.<commit_after>#!/usr/bin/env python

"""
Path-like row selector for pandas DataFrames with hierarchical MultiIndexes.
"""

import re
import numpy as np

import pandas as pd
import ply.lex as lex
import ply.yacc as yacc

class PathLikeSelector(object):
    """
    Class for selecting rows of a pandas DataFrame using path-like selectors.

    Select rows from a pandas DataFrame using path-like selectors.
    Assumes that the DataFrame instance has a MultiIndex where each level
    corresponds to a level of the selector; a level may either be a denoted by a
    string label (e.g., 'foo') or a numerical index (e.g., 0, 1, 2).
    Examples of valid selectors include

    /foo/bar
    /foo/[qux,bar]
    /foo/bar[0]
    /foo/bar[0,1]
    /foo/bar[0:5]
    /foo/*/baz
    /foo/*/baz[5]
    /foo/bar+/baz/qux

    The class can also be used to create new MultiIndex instances from selectors
    that contain no wildcards.

    Notes
    -----
    Numerical indices are assumed to be zero-based. Ranges do not include the
    end element (i.e., like numpy, not like pandas).
    """

    tokens = ('ASTERISK', 'INTEGER', 'INTEGER_SET',
              'INTERVAL', 'STRING', 'STRING_SET', 'PLUS')

    def __init__(self):
        self._setup()

    def _parse_interval_str(self, s):
        """
        Convert string representation of interval to tuple containing numerical
        start and stop values.
        """

        start, stop = s.split(':')
        if start == '':
            start = 0
        else:
            start = int(start)
        if stop == '':
            stop = np.inf
        else:
            stop = int(stop)
        return (start, stop)

    def t_PLUS(self, t):
        r'\+'
        return t

    def t_ASTERISK(self, t):
        r'/\*'
        t.value = t.value.strip('/')
        return t

    def t_INTEGER(self, t):
        r'/?\[\d+\]'
        t.value = int(re.search('\[(\d+)\]', t.value.strip('/')).group(1))
        return t

    def t_INTEGER_SET(self, t):
        r'/?\[(?:\d+,?)+\]'
        t.value = map(int, t.value.strip('[]').split(','))
        return t

    def t_INTERVAL(self, t):
        r'\[\d*\:\d*\]'
        t.value = self._parse_interval_str(re.search('\[(.+)\]', t.value).group(1))
        return t

    def t_STRING(self, t):
        r'/[^*/\[\]:\d][^+*/\[\]:]*'
        t.value = t.value.strip('/')
        return t

    def t_STRING_SET(self, t):
        r'/\[(?:[^+*/\[\]:\d][^+*/\[\]:]*,?)+\]'
        t.value = t.value.strip('/[]').split(',')
        return t

    def t_error(self, t):
        print 'Illegal character "%s"' % t.value[0]
        raise ValueError('Cannot tokenize selector')

    def p_list_plus(self, p):
        'list : list PLUS selector'
        p[0] = p[1]+[p[3]]

    def p_list_selector(self, p):
        'list : selector'
        p[0] = [p[1]]

    def p_selector_plus(self, p):
        'selector : selector token'
        p[0] = p[1]
        p[0].append(p[2])

    def p_selector_token(self, p):
        'selector : token'
        p[0] = [p[1]]

    def p_token(self, p):
        '''token : ASTERISK
               | INTEGER
               | INTEGER_SET
               | INTERVAL
               | STRING
               | STRING_SET'''
        p[0] = p[1]

    def p_error(self, p):
        print 'Syntax error'
        raise ValueError('Cannot parse selector')

    def _setup(self):
        """
        Build lexer and parser.
        """

        self.lexer = lex.lex(module=self)
        self.parser = yacc.yacc(module=self, debug=0)

    def tokenize(self, selector):
        """
        Tokenize a selector string.

        Parameters
        ----------
        selector : str
            Row selector.

        Returns
        -------
        token_list : list
            List of tokens extracted by ply.
        """

        self.lexer.input(selector)
{'fraction_non_alphanumeric': 0.15235885537509666, 'fraction_numerical': 0.006831657643722609, 'mean_word_length': 1.793016558675306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6655715', 'n_tokens_mistral': 2596, 'n_tokens_neox': 2471, 'n_words': 693}