<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim:ts=2:sw=2:expandtab
#
# Copyright (c) 2011, <NAME>. All rights reserved.
#
# <<EMAIL>> <http://nikcub.appspot.com>  
#
# Licensed under a BSD license. You may obtain a copy of the License at
#
#     http://nikcub.appspot.com/bsd-license
#

"""
  Sketch - jinja helpers
  
  * compile templates to python (use in deploy scripts)
  * load compiled templates
  * custom template loader supporting template themes in subdirectories
  * template caching using GAE memcache
  * single enviroment cached across entire app

  @TODO write the custom loader to handle subdirs and subprojects and themes
  @TODO complete compiler
  @TODO option to compile into memcache
  @TODO implement admin interface on compiling
  @TODO monkey patch jinja to load compiled templates: http://paste.pocoo.org/show/124381/
"""

"""
  compile
  ~~~~~~~

  Compiles Jinja2 templates to python code. To compile a whole dir:

    from jinja2 import Environment
    from jinja2_compile import compile_dir

    env = Environment(extensions=['jinja2.ext.i18n'])
    src_path = '/path/to/templates'
    dst_path = '/path/to/templates_compiled'

    compile_dir(env, src_path, dst_path)

  :copyright: (c) 2009 <NAME> <<EMAIL>>.
  :license: BSD, see LICENSE for more details.
"""

import os
import sys
import re
import logging

from floyd.helpers.dateformat import *

try:
  import memcache
except ImportError:
  memcache = {}

try:
  import jinja2
except ImportError:
  logging.info('Could not import Jinja')
  logging.info(sys.path)
  raise NotFound('Could not import jinja2')
    
_jinja_env = None
_jinja_loaders = {}


class JinjaEnviroment(jinja2.Environment):
  def join_path(self, template, parent=None):
    if ':' in template:
      sep, template = template.split(':', 1)
      return "%s:%s/%s" % (sep, parent, template)
    elif '/' in parent:
      return os.path.normpath(os.path.join(parent, '..', template))
    elif parent.endswith('.html'):
      return template
    else:
      return "%s/%s" % (parent, template)
    
    # return template
    # return template
    # return os.path.normpath(os.path.join(parent, '..', template))
    


class MemcacheBytecodeCache(jinja2.BytecodeCache):
  def __init__(self, client=None, prefix='floyd.jinja2.cache.', timeout=None):
    self.client = memcache
    self.prefix = prefix
    self.timeout = timeout

  def load_bytecode(self, bucket):
    code = self.client.get(self.prefix + bucket.key)
    if code is not None:
      bucket.bytecode_from_string(code)

  def dump_bytecode(self, bucket):
    args = (self.prefix + bucket.key, bucket.bytecode_to_string())
    if self.timeout is not None:
      args += (self.timeout,)
    self.client.set(*args)
<commit_msg>cleanup<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim:ts=2:sw=2:expandtab
#
# Copyright (c) 2011, <NAME>. All rights reserved.
#
# <<EMAIL>> <http://nikcub.appspot.com>  
#
# Licensed under a BSD license. You may obtain a copy of the License at
#
#     http://nikcub.appspot.com/bsd-license
#

"""
  Sketch - jinja helpers
  
  * compile templates to python (use in deploy scripts)
  * load compiled templates
  * custom template loader supporting template themes in subdirectories
  * template caching using GAE memcache
  * single enviroment cached across entire app

  @TODO write the custom loader to handle subdirs and subprojects and themes
  @TODO complete compiler
  @TODO option to compile into memcache
  @TODO implement admin interface on compiling
  @TODO monkey patch jinja to load compiled templates: http://paste.pocoo.org/show/124381/
"""

"""
  compile
  ~~~~~~~

  Compiles Jinja2 templates to python code. To compile a whole dir:

    from jinja2 import Environment
    from jinja2_compile import compile_dir

    env = Environment(extensions=['jinja2.ext.i18n'])
    src_path = '/path/to/templates'
    dst_path = '/path/to/templates_compiled'

    compile_dir(env, src_path, dst_path)

  :copyright: (c) 2009 <NAME> <<EMAIL>>.
  :license: BSD, see LICENSE for more details.
"""

import os
import sys
import re
import logging

from floyd.helpers.dateformat import *

try:
  import memcache
except ImportError:
  memcache = {}

try:
  import jinja2
except ImportError:
  logging.info('Could not import Jinja')
  logging.info(sys.path)
  raise NotFound('Could not import jinja2')
    
_jinja_env = None
_jinja_loaders = {}


class JinjaEnviroment(jinja2.Environment):
  def join_path(self, template, parent=None):
    if ':' in template:
      sep, template = template.split(':', 1)
      return "%s:%s/%s" % (sep, parent, template)
    elif '/' in parent:
      return os.path.normpath(os.path.join(parent, '..', template))
    elif parent.endswith('.html'):
      return template

    return "%s/%s" % (parent, template)


class MemcacheBytecodeCache(jinja2.BytecodeCache):
  def __init__(self, client=None, prefix='floyd.jinja2.cache.', timeout=None):
    self.client = memcache
    self.prefix = prefix
    self.timeout = timeout

  def load_bytecode(self, bucket):
    code = self.client.get(self.prefix + bucket.key)
    if code is not None:
      bucket.bytecode_from_string(code)

  def dump_bytecode(self, bucket):
    args = (self.prefix + bucket.key, bucket.bytecode_to_string())
    if self.timeout is not None:
      args += (self.timeout,)
    self.client.set(*args)

{'fraction_non_alphanumeric': 0.11683720930232558, 'fraction_numerical': 0.010790697674418604, 'mean_word_length': 3.5870307167235493, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6927296', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1708, 'n_words': 500}