<commit_before>import os
import re
import time
import cStringIO as StringIO

from template import stash, constants, document, provider, plugins, filters, base, util

DEBUG = None

class Context(base.Base):
  def __init__(self, config):
    base.Base.__init__(self)
    self.LOAD_TEMPLATES = util.listify(config.get("LOAD_TEMPLATES")
                                       or provider.Provider(config))
    self.LOAD_PLUGINS   = util.listify(config.get("LOAD_PLUGINS")
                                       or plugins.Plugins(config))
    self.LOAD_FILTERS   = util.listify(config.get("LOAD_FILTERS")
                                       or filters.Filters(config))
    providers  = self.LOAD_TEMPLATES
    prefix_map = config.get("PREFIX_MAP") or {}
    self.FILTER_CACHE = {}
    self.PREFIX_MAP = {}
    for key, value in prefix_map.items():
      if isinstance(value, list):
        self.PREFIX_MAP[key] = value
      elif isinstance(value, str):
        self.PREFIX_MAP[key] = [providers[x] for x in re.split(r"\D+", value)]
      else:
        self.PREFIX_MAP[key] = value

    # STASH
    if "STASH" in config:
      self.STASH = config["STASH"]
    else:
      predefs = config.get("VARIABLES") or config.get("PRE_DEFINE") or {}
      if predefs.get("_DEBUG") is None:
        debug = config.get("DEBUG", 0) & constants.DEBUG_UNDEF
        predefs["_DEBUG"] = debug and 1 or 0
      self.STASH = stash.Stash(predefs)

    # compile any template BLOCKS specified as text
    self.BLOCKS = {}
    self.BLKSTACK = []
    blocks = config.get("BLOCKS") or {}
    b = {}
    for key, block in blocks.items():
      if isinstance(block, str):
        block = self.template(util.Reference(block))
      b[key] = block
    self.INIT_BLOCKS = self.BLOCKS = b

    self.RECURSION = config.get("RECURSION", False)
    self.EVAL_PYTHON = config.get("EVAL_PYTHON", False)
    self.TRIM = config.get("TRIM", False)
    self.BLKSTACK = []
    self.CONFIG   = config
    if config.get("EXPOSE_BLOCKS") is not None:
      self.EXPOSE_BLOCKS = config.get("EXPOSE_BLOCKS")
    else:
      self.EXPOSE_BLOCKS = False
    self.DEBUG_FORMAT = config.get("DEBUG_FORMAT")
    self.DEBUG_DIRS   = config.get("DEBUG", 0) & constants.DEBUG_DIRS
    if config.get("DEBUG") is not None:
      self.DEBUG = config["DEBUG"] & (constants.DEBUG_CONTEXT |
                                      constants.DEBUG_FLAGS)
    else:
      self.DEBUG = DEBUG

  def config(self):
    return self.CONFIG

  def catch(self, error, output=None):
    if isinstance(error, base.Exception):
      if output:
        error.text(output)
      return error
    else:
      return base.Exception("None", error, output)

  def insert(self, files):
    prefix = None
    providers = None
    text = None
    output = StringIO.StringIO()
    if not isinstance(files, list):
      files = [files]
    for f in files:  # FILE: {
      name = f
      regex = "\w{%d,}:" % (int(os.name == "nt") + 1)
      match = re.match(regex, name)
      if match:
        prefix = match.group(0)
        name = name[len(prefix):]
      if prefix:
        providers = self.PREFIX_MAP.get(prefix)
        if not providers:
          return self.throw(constants.ERROR_FILE,
                            "no providers for file prefix '%s'" % prefix)
      else:
        providers = self.PREFIX_MAP.get("default") or self.LOAD_TEMPLATES

      for provider in providers:
        text, error = provider.load(name, prefix)
        if not error:
          output.write(text)
          break
        if error == constants.STATUS_ERROR:
          if isinstance(text, list):
            self.throw(text)
          else:
            self.throw(constants.ERROR_FILE, text)
      else:
        self.throw(constants.ERROR_FILE, "%s: not found" % f)

    return output.getvalue()

  def throw(self, error, info=None, output=None):
    if isinstance(error, base.Exception):
      raise error
    elif info is not None:
      raise base.Exception(error, info, output)
    else:
      raise base.Exception("None", error or "", output)

  def include(self, template, params=None):
    return self.process(template, params, localize=True)

  def process(self, template, params=None, localize=False):
    output = StringIO.StringIO()
    if not isinstance(template, list):
      template = [template]
    compileds = []
    for name in template:
      compileds.append(self.template(name))
    if localize:
      self.STASH = self.STASH.clone(params)
    else:
      self.STASH.update(params)

    error = None
    try:
      # save current component
      try:
        component = self.STASH.get("component")
      except:
        component = None
      for name, compiled in zip(template, compileds):
        if callable(compiled):
          element = {"modtime": time.time()}
          if isinstance(name, str):
            element["name"] = name
          else:
            element["name"] = ""
        else:
          element = compiled
        if isinstance(component, document.Document):
          if isinstance(element, dict):
            element["caller"] = component.name
            element["callers"] = getattr(component, "callers", [])
            element["callers"].append(element["caller"])
          else:
            element.caller = component.name
            element.callers = getattr(component, "callers", [])
            element.callers.append(element.caller)
        self.STASH.set("component", element)
        if not localize:
          # merge any local blocks defined in the Template::Document
          # info our local BLOCKS cache
          if isinstance(compiled, document.Document):
            tblocks = compiled.blocks()
            if tblocks:
              self.BLOCKS.update(tblocks)
        if callable(compiled):
          tmpout = compiled(self)
        elif util.can(compiled, "process"):
          tmpout = compiled.process(self)
        else:
          self.throw("file", "invalid template reference: %s" % compiled)
        if self.TRIM:
          tmpout = tmpout.strip()
        output.write(tmpout)
        # pop last item from callers
        if isinstance(component, document.Document):
          if isinstance(element, dict):
            element["callers"].pop()
          else:
            element.callers.pop()
      self.STASH.set("component", component)
    except base.Exception, e:
      error = e

    if localize:<commit_msg>Bugfix.<commit_after>import os
import re
import time
import cStringIO as StringIO

from template import stash, constants, document, provider, plugins, filters, base, util

DEBUG = None

class Context(base.Base):
  def __init__(self, config):
    base.Base.__init__(self)
    self.LOAD_TEMPLATES = util.listify(config.get("LOAD_TEMPLATES")
                                       or provider.Provider(config))
    self.LOAD_PLUGINS   = util.listify(config.get("LOAD_PLUGINS")
                                       or plugins.Plugins(config))
    self.LOAD_FILTERS   = util.listify(config.get("LOAD_FILTERS")
                                       or filters.Filters(config))
    providers  = self.LOAD_TEMPLATES
    prefix_map = config.get("PREFIX_MAP") or {}
    self.FILTER_CACHE = {}
    self.PREFIX_MAP = {}
    for key, value in prefix_map.items():
      if isinstance(value, list):
        self.PREFIX_MAP[key] = value
      elif isinstance(value, str):
        self.PREFIX_MAP[key] = [providers[x] for x in re.split(r"\D+", value)]
      else:
        self.PREFIX_MAP[key] = value

    # STASH
    if "STASH" in config:
      self.STASH = config["STASH"]
    else:
      predefs = config.get("VARIABLES") or config.get("PRE_DEFINE") or {}
      if predefs.get("_DEBUG") is None:
        debug = config.get("DEBUG", 0) & constants.DEBUG_UNDEF
        predefs["_DEBUG"] = debug and 1 or 0
      self.STASH = stash.Stash(predefs)

    # compile any template BLOCKS specified as text
    self.BLOCKS = {}
    self.BLKSTACK = []
    blocks = config.get("BLOCKS") or {}
    b = {}
    for key, block in blocks.items():
      if isinstance(block, str):
        block = self.template(util.Reference(block))
      b[key] = block
    self.INIT_BLOCKS = self.BLOCKS = b

    self.RECURSION = config.get("RECURSION", False)
    self.EVAL_PYTHON = config.get("EVAL_PYTHON", False)
    self.TRIM = config.get("TRIM", False)
    self.BLKSTACK = []
    self.CONFIG   = config
    if config.get("EXPOSE_BLOCKS") is not None:
      self.EXPOSE_BLOCKS = config.get("EXPOSE_BLOCKS")
    else:
      self.EXPOSE_BLOCKS = False
    self.DEBUG_FORMAT = config.get("DEBUG_FORMAT")
    self.DEBUG_DIRS   = config.get("DEBUG", 0) & constants.DEBUG_DIRS
    if config.get("DEBUG") is not None:
      self.DEBUG = config["DEBUG"] & (constants.DEBUG_CONTEXT |
                                      constants.DEBUG_FLAGS)
    else:
      self.DEBUG = DEBUG

  def config(self):
    return self.CONFIG

  def catch(self, error, output=None):
    if isinstance(error, base.Exception):
      if output:
        error.text(output)
      return error
    else:
      return base.Exception("None", error, output)

  def insert(self, files):
    prefix = None
    providers = None
    text = None
    output = StringIO.StringIO()
    if not isinstance(files, list):
      files = [files]
    for f in files:  # FILE: {
      name = f
      regex = "\w{%d,}:" % (int(os.name == "nt") + 1)
      match = re.match(regex, name)
      if match:
        prefix = match.group(0)
        name = name[len(prefix):]
      if prefix:
        providers = self.PREFIX_MAP.get(prefix)
        if not providers:
          return self.throw(constants.ERROR_FILE,
                            "no providers for file prefix '%s'" % prefix)
      else:
        providers = self.PREFIX_MAP.get("default") or self.LOAD_TEMPLATES

      for provider in providers:
        text, error = provider.load(name, prefix)
        if not error:
          output.write(text)
          break
        if error == constants.STATUS_ERROR:
          if isinstance(text, list):
            self.throw(text)
          else:
            self.throw(constants.ERROR_FILE, text)
      else:
        self.throw(constants.ERROR_FILE, "%s: not found" % f)

    return output.getvalue()

  def throw(self, error, info=None, output=None):
    if isinstance(error, base.Exception):
      raise error
    elif info is not None:
      raise base.Exception(error, info, output)
    else:
      raise base.Exception("None", error or "", output)

  def include(self, template, params=None):
    return self.process(template, params, localize=True)

  def process(self, template, params=None, localize=False):
    output = StringIO.StringIO()
    if not isinstance(template, list):
      template = [template]
    compileds = []
    for name in template:
      compileds.append(self.template(name))
    if localize:
      self.STASH = self.STASH.clone(params)
    else:
      self.STASH.update(params)

    error = None
    try:
      # save current component
      try:
        component = self.STASH.get("component")
      except:
        component = None
      for name, compiled in zip(template, compileds):
        if callable(compiled):
          element = {"modtime": time.time()}
          if isinstance(name, str):
            element["name"] = name
          else:
            element["name"] = ""
        else:
          element = compiled
        if isinstance(component, document.Document):
          if isinstance(element, dict):
            element["caller"] = component.name
            element["callers"] = getattr(component, "callers") or []
            element["callers"].append(element["caller"])
          else:
            element.caller = component.name
            element.callers = getattr(component, "callers") or []
            element.callers.append(element.caller)
        self.STASH.set("component", element)
        if not localize:
          # merge any local blocks defined in the Template::Document
          # info our local BLOCKS cache
          if isinstance(compiled, document.Document):
            tblocks = compiled.blocks()
            if tblocks:
              self.BLOCKS.update(tblocks)
        if callable(compiled):
          tmpout = compiled(self)
        elif util.can(compiled, "process"):
          tmpout = compiled.process(self)
        else:
          self.throw("file", "invalid template reference: %s" % compiled)
        if self.TRIM:
          tmpout = tmpout.strip()
        output.write(tmpout)
        # pop last item from callers
        if isinstance(component, document.Document):
          if isinstance(element, dict):
            element["callers"].pop()
          else:
            element.callers.pop()
      self.STASH.set("component", component)
    except base.Exception, e:
      error = e

    if localize:
{'fraction_non_alphanumeric': 0.11499374609130707, 'fraction_numerical': 0.0009380863039399625, 'mean_word_length': 2.228924785461888, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7178930', 'n_tokens_mistral': 3773, 'n_tokens_neox': 3723, 'n_words': 919}