<commit_before>import ast
import operator
import re
import itertools
import collections
import ibis

from . import utils
from . import filters
from . import errors


# Dictionary of registered keywords for instruction tags.
instruction_keywords = {}


# List of registered endwords for instruction tags with block scope.
instruction_endwords = []


# Decorator function for registering handler classes for instruction tags.
# Registering an endword gives the instruction tag block scope.
def register(keyword, endword=None):

    def register_node_class(node_class):
        instruction_keywords[keyword] = (node_class, endword)
        if endword:
            instruction_endwords.append(endword)
        return node_class

    return register_node_class


# Helper class for evaluating expression strings.
#
# An Expression object is initialized with an expression string parsed from a template. An
# expression string can contain a variable name or a Python literal, optionally followed by a
# sequence of filters.
#
# The Expression object handles the rather convoluted process of parsing the string, evaluating
# the literal or resolving the variable, calling the variable if it resolves to a callable, and
# applying the filters to the resulting object. The consumer simply needs to call the expression's
# .eval() method and supply an appropriate Context object.
#
# Examples of valid expression syntax include:
#
#     foo.bar.baz|default('bam')|escape
#     'foo', 'bar', 'baz'|random
#
# Arguments can be passed to callables using bracket syntax:
#
#     foo.bar.baz('bam')|filter(25, 'text')
#
class Expression:

    re_func_call = re.compile(r'^([a-zA-Z_][a-zA-Z0-9_.]*)[(](.*)[)]$')

    def __init__(self, expr, token):
        self.token = token
        self.filters = []
        pipe_split = utils.splitc(expr.strip(), '|', strip=True)
        self._parse_primary_expr(pipe_split[0])
        self._parse_filters(pipe_split[1:])
        if self.is_literal:
            self.literal = self._apply_filters_to_literal(self.literal)

    def _parse_primary_expr(self, expr):
        try:
            self.literal = ast.literal_eval(expr)
            self.is_literal = True
        except:
            self.is_func_call, self.varstring, self.func_args = self._try_parse_as_func_call(expr)
            self.is_literal = False

    def _try_parse_as_func_call(self, expr):
        match = self.re_func_call.match(expr)
        if not match:
            return False, expr, []
        func_name = match.group(1)
        func_args = utils.splitc(match.group(2), ',', True, True)
        for index, arg in enumerate(func_args):
            try:
                func_args[index] = ast.literal_eval(arg)
            except Exception as err:
                msg = f"Unparsable argument '{arg}' in template '{self.token.template_id}', "
                msg += f"line {self.token.line_number}. "
                msg += f"Arguments must be valid Python literals."
                raise errors.TemplateSyntaxError(msg, self.token) from err
        return True, func_name, func_args

    def _parse_filters(self, filter_list):
        for filter_expr in filter_list:
            _, filter_name, filter_args = self._try_parse_as_func_call(filter_expr)
            if filter_name in filters.filtermap:
                self.filters.append((filter_name, filters.filtermap[filter_name], filter_args))
            else:
                msg = f"Unrecognised filter name '{filter_name}' in template '{self.token.template_id}', "
                msg += f"line {self.token.line_number}."
                raise errors.TemplateSyntaxError(msg, self.token)

    def _apply_filters_to_literal(self, obj):
        for name, func, args in self.filters:
            try:
                obj = func(obj, *args)
            except Exception as err:
                msg = f"Error applying filter '{name}' to literal "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}."
                raise errors.TemplateSyntaxError(msg, self.token) from err
        return obj

    def eval(self, context):
        if self.is_literal:
            return self.literal
        else:
            return self._resolve_variable(context)

    def _resolve_variable(self, context):
        obj = context.resolve(self.varstring, self.token)
        if self.is_func_call:
            try:
                obj = obj(*self.func_args)
            except Exception as err:
                msg = f"Error calling function '{self.varstring}' "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}."
                raise errors.TemplateRenderingError(msg, self.token) from err
        return self._apply_filters_to_variable(obj)

    def _apply_filters_to_variable(self, obj):
        for name, func, args in self.filters:
            try:
                obj = func(obj, *args)
            except Exception as err:
                msg = f"Error applying filter '{name}' to variable "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}."
                raise errors.TemplateRenderingError(msg, self.token) from err
        return obj


# Base class for all node objects. To render a node into a string call its .render() method.
# Subclasses shouldn't override the base .render() method; instead they should override
# .wrender() which ensures that any uncaught exceptions are wrapped in a TemplateRenderingError.
class Node:

    def __init__(self, token=None, children=None):
        self.token = token
        self.children = children or []
        try:
            self.process_token(token)
        except errors.TemplateError:
            raise
        except Exception as err:
            if token:
                tagname = f"'{token.keyword}'" if token.type == "INSTRUCTION" else token.type
                msg = f"An unexpected error occurred while parsing the {tagname} tag "
                msg += f"in template '{token.template_id}', line {token.line_number}: "
                msg += f"{err.__class__.__name__}: {err}"
            else:
                msg = f"Unexpected syntax error: {err.__class__.__name__}: {err}"
            raise errors.TemplateSyntaxError(msg, token) from err

    def __str__(self):
        return self.to_str()

    def to_str(self, depth=0):
        output = ["·  " * depth + f"{self.__class__.__name__}"]
        for child in self.children:
            output.append(child.to_str(depth + 1))
        return "\n".join(output)

    def render(self, context):
        try:
            return self.wrender(context)
        except errors.TemplateError:
            raise
        except Exception as err:
            if self.token:
                tagname = f"'{self.token.keyword}'" if self.token.type == "INSTRUCTION" else self.token.type
                msg = f"An unexpected error occurred while rendering the {tagname} tag "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}: "
                msg += f"{err.__class__.__name__}: {err}"
            else:
                msg = f"Unexpected rendering error: {err.__class__.__name__}: {err}"
            raise errors.TemplateRenderingError(msg, None) from err

    def wrender(self, context):
        return ''.join(child.render(context) for child in self.children)<commit_msg>Fix missing token<commit_after>import ast
import operator
import re
import itertools
import collections
import ibis

from . import utils
from . import filters
from . import errors


# Dictionary of registered keywords for instruction tags.
instruction_keywords = {}


# List of registered endwords for instruction tags with block scope.
instruction_endwords = []


# Decorator function for registering handler classes for instruction tags.
# Registering an endword gives the instruction tag block scope.
def register(keyword, endword=None):

    def register_node_class(node_class):
        instruction_keywords[keyword] = (node_class, endword)
        if endword:
            instruction_endwords.append(endword)
        return node_class

    return register_node_class


# Helper class for evaluating expression strings.
#
# An Expression object is initialized with an expression string parsed from a template. An
# expression string can contain a variable name or a Python literal, optionally followed by a
# sequence of filters.
#
# The Expression object handles the rather convoluted process of parsing the string, evaluating
# the literal or resolving the variable, calling the variable if it resolves to a callable, and
# applying the filters to the resulting object. The consumer simply needs to call the expression's
# .eval() method and supply an appropriate Context object.
#
# Examples of valid expression syntax include:
#
#     foo.bar.baz|default('bam')|escape
#     'foo', 'bar', 'baz'|random
#
# Arguments can be passed to callables using bracket syntax:
#
#     foo.bar.baz('bam')|filter(25, 'text')
#
class Expression:

    re_func_call = re.compile(r'^([a-zA-Z_][a-zA-Z0-9_.]*)[(](.*)[)]$')

    def __init__(self, expr, token):
        self.token = token
        self.filters = []
        pipe_split = utils.splitc(expr.strip(), '|', strip=True)
        self._parse_primary_expr(pipe_split[0])
        self._parse_filters(pipe_split[1:])
        if self.is_literal:
            self.literal = self._apply_filters_to_literal(self.literal)

    def _parse_primary_expr(self, expr):
        try:
            self.literal = ast.literal_eval(expr)
            self.is_literal = True
        except:
            self.is_func_call, self.varstring, self.func_args = self._try_parse_as_func_call(expr)
            self.is_literal = False

    def _try_parse_as_func_call(self, expr):
        match = self.re_func_call.match(expr)
        if not match:
            return False, expr, []
        func_name = match.group(1)
        func_args = utils.splitc(match.group(2), ',', True, True)
        for index, arg in enumerate(func_args):
            try:
                func_args[index] = ast.literal_eval(arg)
            except Exception as err:
                msg = f"Unparsable argument '{arg}' in template '{self.token.template_id}', "
                msg += f"line {self.token.line_number}. "
                msg += f"Arguments must be valid Python literals."
                raise errors.TemplateSyntaxError(msg, self.token) from err
        return True, func_name, func_args

    def _parse_filters(self, filter_list):
        for filter_expr in filter_list:
            _, filter_name, filter_args = self._try_parse_as_func_call(filter_expr)
            if filter_name in filters.filtermap:
                self.filters.append((filter_name, filters.filtermap[filter_name], filter_args))
            else:
                msg = f"Unrecognised filter name '{filter_name}' in template '{self.token.template_id}', "
                msg += f"line {self.token.line_number}."
                raise errors.TemplateSyntaxError(msg, self.token)

    def _apply_filters_to_literal(self, obj):
        for name, func, args in self.filters:
            try:
                obj = func(obj, *args)
            except Exception as err:
                msg = f"Error applying filter '{name}' to literal "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}."
                raise errors.TemplateSyntaxError(msg, self.token) from err
        return obj

    def eval(self, context):
        if self.is_literal:
            return self.literal
        else:
            return self._resolve_variable(context)

    def _resolve_variable(self, context):
        obj = context.resolve(self.varstring, self.token)
        if self.is_func_call:
            try:
                obj = obj(*self.func_args)
            except Exception as err:
                msg = f"Error calling function '{self.varstring}' "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}."
                raise errors.TemplateRenderingError(msg, self.token) from err
        return self._apply_filters_to_variable(obj)

    def _apply_filters_to_variable(self, obj):
        for name, func, args in self.filters:
            try:
                obj = func(obj, *args)
            except Exception as err:
                msg = f"Error applying filter '{name}' to variable "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}."
                raise errors.TemplateRenderingError(msg, self.token) from err
        return obj


# Base class for all node objects. To render a node into a string call its .render() method.
# Subclasses shouldn't override the base .render() method; instead they should override
# .wrender() which ensures that any uncaught exceptions are wrapped in a TemplateRenderingError.
class Node:

    def __init__(self, token=None, children=None):
        self.token = token
        self.children = children or []
        try:
            self.process_token(token)
        except errors.TemplateError:
            raise
        except Exception as err:
            if token:
                tagname = f"'{token.keyword}'" if token.type == "INSTRUCTION" else token.type
                msg = f"An unexpected error occurred while parsing the {tagname} tag "
                msg += f"in template '{token.template_id}', line {token.line_number}: "
                msg += f"{err.__class__.__name__}: {err}"
            else:
                msg = f"Unexpected syntax error: {err.__class__.__name__}: {err}"
            raise errors.TemplateSyntaxError(msg, token) from err

    def __str__(self):
        return self.to_str()

    def to_str(self, depth=0):
        output = ["·  " * depth + f"{self.__class__.__name__}"]
        for child in self.children:
            output.append(child.to_str(depth + 1))
        return "\n".join(output)

    def render(self, context):
        try:
            return self.wrender(context)
        except errors.TemplateError:
            raise
        except Exception as err:
            if self.token:
                tagname = f"'{self.token.keyword}'" if self.token.type == "INSTRUCTION" else self.token.type
                msg = f"An unexpected error occurred while rendering the {tagname} tag "
                msg += f"in template '{self.token.template_id}', line {self.token.line_number}: "
                msg += f"{err.__class__.__name__}: {err}"
            else:
                msg = f"Unexpected rendering error: {err.__class__.__name__}: {err}"
            raise errors.TemplateRenderingError(msg, self.token) from err

    def wrender(self, context):
        return ''.join(child.render(context) for child in self.children)
{'fraction_non_alphanumeric': 0.09621365787694389, 'fraction_numerical': 0.001352265043948614, 'mean_word_length': 2.4429702048417132, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7168476', 'n_tokens_mistral': 4113, 'n_tokens_neox': 3999, 'n_words': 1209}