<commit_before>from .utils import *
from mako.template import Template
import pickle
import yaml
import hashlib
import logging



loader = pickle.loads
dumper = pickle.dumps
#loader = yaml.load
#dumper = yaml.dump


def renderTree( data, imports = None, strict_undefined = True, filter = toNum ):
  '''Renders a nested data structure with itself.

     Given a data tree that contains some parameters containing references to
     other parmaeters,

     param2 : ${param1}

     this function will replace parameter references with thier values.
     Mako Templates are used for the rendering with the following steps
     
      - the data tree is written to a data string
      - the data string loaded as a Mako template
      - the data string s rendered using the data tree as the context
      - the data string is decoded back to a data tree, replacing the original

     These steps repeated until a the data string is unchanged by a render.
     '''

  # dump the tree to a string
  serialized_data = dumper(data)
  # keep rendering until the data string repeats itself.
  hashes = dict()
  hasher = hashlib.sha1
  hash = hasher(serialized_data).hexdigest()
  # make sure the numbers are numbers
  data = applyFilter(data, filter)
  while hashes.get( hash, 0 ) < 2:
    # run the string through a Mako template using the tree for context
    logging.debug("RENDERING (%s)" % hash)
    logging.debug( serialized_data )
    logging.debug( data )
    t = Template(serialized_data, imports=imports, ignore_expression_errors=True)
    serialized_data = t.render( **toAttrDict(data) )
    data = loader( serialized_data )
    # turn everything we can into a number and update output_text
    data = applyFilter(data, filter)
    serialized_data = dumper(data)
    hash = hasher(serialized_data).hexdigest()
    hashes[hash] = hashes.get(hash,0) + 1
    logging.debug("RENDERED (%s)" % hash)
    logging.debug( serialized_data )


  # run make one more time with strict errors so that we will get
  # an exception if there are still any expressions left
  if strict_undefined:
    t = Template(input_text, strict_undefined=True, ignore_expression_errors=False)
    output_text = t.render( )

  return data

def scopedRenderTree( data, imports = None, strict_undefined = True, filter = toNum ):
  '''Render a data tree by applying the renderTree function to each branch, starting from the bottom.

     This function also acts recursivly. It 'walks' down the data tree until
     it reaches the end of a branch. The renderTree function is applied to each branch<commit_msg>BUGFIX: support for lists<commit_after>from .utils import *
from mako.template import Template
import pickle
import yaml
import hashlib
import logging



loader = pickle.loads
dumper = pickle.dumps
#loader = yaml.load
#dumper = yaml.dump


def renderTree( data, imports = None, strict_undefined = True, filter = toNum ):
  '''Renders a nested data structure with itself.

     Given a data tree that contains some parameters containing references to
     other parmaeters,

     param2 : ${param1}

     this function will replace parameter references with thier values.
     Mako Templates are used for the rendering with the following steps
     
      - the data tree is written to a data string
      - the data string loaded as a Mako template
      - the data string s rendered using the data tree as the context
      - the data string is decoded back to a data tree, replacing the original

     These steps repeated until a the data string is unchanged by a render.
     '''

  # dump the tree to a string
  serialized_data = dumper(data)
  # keep rendering until the data string repeats itself.
  hashes = dict()
  hasher = hashlib.sha1
  hash = hasher(serialized_data).hexdigest()
  # make sure the numbers are numbers
  data = applyFilter(data, filter)
  while hashes.get( hash, 0 ) < 2:
    # run the string through a Mako template using the tree for context
    logging.debug("RENDERING (%s)" % hash)
    logging.debug( serialized_data )
    logging.debug( data )
    t = Template(serialized_data, imports=imports, ignore_expression_errors=True)
    if isinstance(data,dict):
      data = toAttrDict(data)
    else:
      data = {'this' : data }
    serialized_data = t.render( **data )
    data = loader( serialized_data )
    # turn everything we can into a number and update output_text
    data = applyFilter(data, filter)
    serialized_data = dumper(data)
    hash = hasher(serialized_data).hexdigest()
    hashes[hash] = hashes.get(hash,0) + 1
    logging.debug("RENDERED (%s)" % hash)
    logging.debug( serialized_data )


  # run make one more time with strict errors so that we will get
  # an exception if there are still any expressions left
  if strict_undefined:
    t = Template(input_text, strict_undefined=True, ignore_expression_errors=False)
    output_text = t.render( )

  return data

def scopedRenderTree( data, imports = None, strict_undefined = True, filter = toNum ):
  '''Render a data tree by applying the renderTree function to each branch, starting from the bottom.

     This function also acts recursivly. It 'walks' down the data tree until
     it reaches the end of a branch. The renderTree function is applied to each branch
{'fraction_non_alphanumeric': 0.068, 'fraction_numerical': 0.0026666666666666666, 'mean_word_length': 3.495719178082192, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5864749', 'n_tokens_mistral': 1468, 'n_tokens_neox': 1428, 'n_words': 616}