<commit_before>"""
The magics offered by the holoviews IPython extension are powerful and
support rich, compositional specifications. To avoid the the brittle,
convoluted code that results from trying to support the syntax in pure
Python, this file defines suitable parsers using pyparsing that are
cleaner and easier to understand.

Pyparsing is required by matplotlib and will therefore be available if
holoviews is being used in conjunction with matplotlib.
"""

import string
from holoviews.core.options import Options

from itertools import groupby
import pyparsing as pp

from ..operation import Compositor
from ..plotting import Plot

ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!#$%&\()*+,-./:;<=>?@\\^_`{|}~'


class Parser(object):
    """
    Base class for magic line parsers, designed for forgiving parsing
    of keyword lists.
    """

    namespace = {}

    @classmethod
    def _strip_commas(cls, kw):
        "Strip out any leading/training commas from the token"
        kw = kw[:-1] if kw[-1]==',' else kw
        return kw[1:] if kw[0]==',' else kw

    @classmethod
    def collect_tokens(cls, parseresult, mode):
        """
        Collect the tokens from a (potentially) nested parse result.
        """
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                tokens[-1] = tokens[-1] + (inner % ''.join(token))
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens

    @classmethod
    def todict(cls, parseresult, mode='parens'):
        """
        Helper function to return dictionary given the parse results
        from a pyparsing.nestedExpr object (containing keywords).
        """
        grouped, kwargs = [], {}
        tokens = cls.collect_tokens(parseresult, mode)
        # Group tokens without '=' and append to last token containing '='
        for group in groupby(tokens, lambda el: '=' in el):
            (val, items) = group
            if val is True:
                grouped += [el for el in items]
            if val is False:
                grouped[-1] += ''.join(items)

        for keyword in grouped:
            try:     kwargs.update(eval('dict(%s)' % keyword, cls.namespace))
            except:  raise SyntaxError("Could not evaluate keyword: %r" % keyword)
        return kwargs



class OptsSpec(Parser):
    """
    An OptsSpec is a string specification that describes an
    OptionTree. It is a list of tree path specifications (using dotted
    syntax) separated by keyword lists for any of the normalization,
    plotting options or style options (in braces, square brackets and
    parentheses respectively). All these option sets are optional and
    may be supplied in any order.

    For instance, the following string:

    Matrix (interpolation='nearest') [show_title=False] Curve color='r'

    Would specify an OptionTree with Options(show_title=False) plot
    options for Matrix and style options Options(color='r') for
    Curve.

    The parser is fairly forgiving; commas between keywords are
    optional and additional spaces are often allowed. The only
    restriction is that keywords *must* be immediately followed by the
    '=' sign (no space).
    """

    plot_options = pp.nestedExpr('[',
                                 ']',
                                 content=pp.OneOrMore(pp.Word(allowed) ^ pp.quotedString)
                             ).setResultsName('plot_options')

    style_options = pp.nestedExpr(opener='(',
                                  closer=')',
                                  ignoreExpr=None
                              ).setResultsName("style_options")

    norm_options = pp.nestedExpr(opener='{',
                                 closer='}',
                                 ignoreExpr=None
                             ).setResultsName("norm_options")


    compositor_ops = pp.MatchFirst(
        [pp.Literal(el.value) for el in Compositor.definitions])

    dotted_path = pp.Combine( pp.Word(ascii_uppercase, exact=1)
                              + pp.Word(pp.alphanums+'._'))


    pathspec = (dotted_path | compositor_ops).setResultsName("pathspec")


    spec_group = pp.Group(pathspec
                          & pp.Optional(norm_options)
                          & pp.Optional(plot_options)
                          & pp.Optional(style_options))

    opts_spec = pp.OneOrMore(spec_group)


    @classmethod
    def process_normalization(cls, parse_group):
        """
        Given a normalization parse group (i.e. the contents of the
        braces), validate the option list and compute the appropriate
        integer value for the normalization plotting option.
        """
        if ('norm_options' not in parse_group): return None
        opts = parse_group['norm_options'][0].asList()
        if opts == []: return None
<commit_msg>OptsSpec parser now allows explicit plot, style and norm keyword lists<commit_after>"""
The magics offered by the holoviews IPython extension are powerful and
support rich, compositional specifications. To avoid the the brittle,
convoluted code that results from trying to support the syntax in pure
Python, this file defines suitable parsers using pyparsing that are
cleaner and easier to understand.

Pyparsing is required by matplotlib and will therefore be available if
holoviews is being used in conjunction with matplotlib.
"""

import string
from holoviews.core.options import Options

from itertools import groupby
import pyparsing as pp

from ..operation import Compositor
from ..plotting import Plot

ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!#$%&\()*+,-./:;<=>?@\\^_`{|}~'


class Parser(object):
    """
    Base class for magic line parsers, designed for forgiving parsing
    of keyword lists.
    """

    namespace = {}

    @classmethod
    def _strip_commas(cls, kw):
        "Strip out any leading/training commas from the token"
        kw = kw[:-1] if kw[-1]==',' else kw
        return kw[1:] if kw[0]==',' else kw

    @classmethod
    def collect_tokens(cls, parseresult, mode):
        """
        Collect the tokens from a (potentially) nested parse result.
        """
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                tokens[-1] = tokens[-1] + (inner % ''.join(token))
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens

    @classmethod
    def todict(cls, parseresult, mode='parens'):
        """
        Helper function to return dictionary given the parse results
        from a pyparsing.nestedExpr object (containing keywords).
        """
        grouped, kwargs = [], {}
        tokens = cls.collect_tokens(parseresult, mode)
        # Group tokens without '=' and append to last token containing '='
        for group in groupby(tokens, lambda el: '=' in el):
            (val, items) = group
            if val is True:
                grouped += [el for el in items]
            if val is False:
                grouped[-1] += ''.join(items)

        for keyword in grouped:
            try:     kwargs.update(eval('dict(%s)' % keyword, cls.namespace))
            except:  raise SyntaxError("Could not evaluate keyword: %r" % keyword)
        return kwargs



class OptsSpec(Parser):
    """
    An OptsSpec is a string specification that describes an
    OptionTree. It is a list of tree path specifications (using dotted
    syntax) separated by keyword lists for any of the style, plotting
    or normalization options. These keyword lists are denoted
    'plot(..)', 'style(...)' and 'norm(...)'  respectively.  These
    three groups may be specified even more concisely using keyword
    lists delimited by square brackets, parentheses and braces
    respectively.  All these sets are optional and may be supplied in
    any order.

    For instance, the following string:

    Matrix (interpolation=None) plot(show_title=False) Curve style(color='r')

    Would specify an OptionTree where Matrix has "interpolation=None"
    for style and 'show_title=False' for plot options. The Curve has a
    style set such that color='r'.

    The parser is fairly forgiving; commas between keywords are
    optional and additional spaces are often allowed. The only
    restriction is that keywords *must* be immediately followed by the
    '=' sign (no space).
    """

    plot_options_short = pp.nestedExpr('[',
                                       ']',
                                       content=pp.OneOrMore(pp.Word(allowed) ^ pp.quotedString)
                                   ).setResultsName('plot_options')

    plot_options_long = pp.nestedExpr('plot(',
                                      ')',
                                      ignoreExpr=None,
                                  ).setResultsName('plot_options')

    plot_options = (plot_options_short | plot_options_long)

    style_options_short = pp.nestedExpr(opener='(',
                                        closer=')',
                                        ignoreExpr=None
                                    ).setResultsName("style_options")

    style_options_long = pp.nestedExpr(opener='style(',
                                       closer=')',
                                       ignoreExpr=None
                                   ).setResultsName("style_options")

    style_options = (style_options_short | style_options_long)


    norm_options_short = pp.nestedExpr(opener='{',
                                       closer='}',
                                       ignoreExpr=None
                                   ).setResultsName("norm_options")

    norm_options_long = pp.nestedExpr(opener='norm(',
                                      closer=')',
                                      ignoreExpr=None
                                  ).setResultsName("norm_options")

    norm_options = (norm_options_short | norm_options_long)

    compositor_ops = pp.MatchFirst(
        [pp.Literal(el.value) for el in Compositor.definitions])

    dotted_path = pp.Combine( pp.Word(ascii_uppercase, exact=1)
                              + pp.Word(pp.alphanums+'._'))


    pathspec = (dotted_path | compositor_ops).setResultsName("pathspec")


    spec_group = pp.Group(pathspec
                          & pp.Optional(norm_options)
                          & pp.Optional(plot_options)
                          & pp.Optional(style_options))

    opts_spec = pp.OneOrMore(spec_group)


    @classmethod
    def process_normalization(cls, parse_group):
        """
        Given a normalization parse group (i.e. the contents of the
        braces), validate the option list and compute the appropriate
        integer value for the normalization plotting option.
        """
        if ('norm_options' not in parse_group): return None
        opts = parse_group['norm_options'][0].asList()
        if opts == []: return None

{'fraction_non_alphanumeric': 0.0913983161183925, 'fraction_numerical': 0.003298324798194601, 'mean_word_length': 2.115738236884803, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4482893', 'n_tokens_mistral': 3127, 'n_tokens_neox': 2978, 'n_words': 984}