<commit_before>
import os
import sys
import shutil
import shelve
import json
import re
from tempfile import mkdtemp
from datetime import datetime
from cement.core.controller import CementBaseController, expose
from cement.utils import fs, shell
from boss import VERSION
from boss.core import exc as boss_exc
    
if sys.version_info[0] < 3:
    from urllib2 import urlopen, HTTPError # pragma: no cover
else:
    from urllib.request import urlopen, HTTPError # pragma: no cover
    
    
BANNER = """

  _______  _______ _______ _______ 
 |   _   \|   _   |   _   |   _   |
 |.  1   /|.  |   |   1___|   1___|
 |.  _   \|.  |   |____   |____   |
 |:  1    |:  1   |:  1   |:  1   |
 |::.. .  |::.. . |::.. . |::.. . |
 `-------'`-------`-------`-------'
               (c) 2012 <NAME>
                           v%s
                           
""" % VERSION

class Template(object):
    def __init__(self, app, path):
        self.app = app
        self.basedir = fs.abspath(path)
        self.config = self._get_config()
        self._word_map = dict()
        self._vars = dict()
        
    def _get_config(self):
        config = None
        possibles = [
            fs.abspath(os.path.join(self.basedir, 'boss.json')),
            fs.abspath(os.path.join(self.basedir, 'boss.yml')),
            ]
        for path in possibles:
            if os.path.exists(path):
                if os.path.basename(path) == 'boss.json':
                    config = self._get_json_config(path)
                    break
                elif os.path.basename(path) == 'boss.yml':
                    config = self._get_yaml_config(path)
                    break
        
        if not config:
            raise boss_exc.BossTemplateError("No supported config found.")
                
        # fix it up with some defaults
        if not config.has_key('delimiter'):
            config['delimiter'] = '@'
        
        return config
        
    def _get_json_config(self, path):
        full_path = fs.abspath(path)
        if not os.path.exists(full_path):
            raise boss_exc.BossTemplateError("Invalid template config.")
        
        self.app.log.debug('loading template config %s' % full_path)
                    
        import json
        return json.load(open(full_path, 'r'))
        
    def _get_yaml_config(self, path):
        full_path = fs.abspath(path)
        if not os.path.exists(full_path):
            raise boss_exc.BossTemplateError("Invalid template config.")
        
        self.app.log.debug('loading template config %s' % full_path)
                    
        try:
            import yaml
        except ImportError as e:
            raise boss_exc.BossRuntimeError("Unable to import yaml.  " + 
                                            "Please install pyYaml.")
                
        return yaml.load(open(full_path, 'r'))
        
    def _populate_vars(self):
        if self.config.has_key('variables'):
            for var,question in self.config['variables'].items():
                if self.app.config.has_key('answers', var.lower()):
                    default = self.app.config.get('answers', var.lower())
                    res = raw_input("%s: [%s] " % (question, default))
                    if len(res) == 0:
                        res = default
                else:
                    res = raw_input("%s: " % question)
                self._vars[var] = res.strip()
                
    def _sub(self, txt):
        # do per item substitution rather than entire txt to avoid variable
        # confusion.  Also allows us to call .capitalize, .lower, etc on the
        # string after substitution.
        for line in str(txt).split('\n'):
            for item in line.split(' '):
                # Not a template var? (generic pattern)
                pattern = "(.*)\%s(.*)\%s(.*)" % \
                    (self.config['delimiter'], self.config['delimiter'])
                    
                if not re.match(pattern, item):
                    continue
            
                for key,value in sorted(self._vars.items()):
                    if key in self._word_map:
                        continue
                    
                    pattern = "(.*)\%s(%s)\%s(.*)" % (
                        self.config['delimiter'], 
                        key, 
                        self.config['delimiter'],
                        )
                         
                    m = re.match(pattern, item)
                    if m:
                        map_key = "%s%s%s" % (
                            self.config['delimiter'], 
                            m.group(2),
                            self.config['delimiter'],
                            )
                        self._word_map[map_key] = str(value)
                    else:
                        pattern = "(.*)\%s(%s)\.([_a-z0-9]*)\%s(.*)" % (
                            self.config['delimiter'], 
                            key,
                            self.config['delimiter'],
                            )
                        m = re.match(pattern, item)
                        if m:
                            if len(m.group(3)) > 0:
                                fixed = str(getattr(value, m.group(3))())
                            else:
                                fixed = str(value)

                            new_key = "%s%s.%s%s" % (
                                self.config['delimiter'], 
                                m.group(2), 
                                m.group(3),
                                self.config['delimiter'],
                                )
                            self._word_map[new_key] = fixed
        
        # actually replace the text
        new_txt = txt
        for pattern,replacement in sorted(self._word_map.items()):
            new_txt = re.sub(pattern, replacement, new_txt)
            
        return new_txt<commit_msg>Resolved Issue #8<commit_after>
import os
import sys
import shutil
import shelve
import json
import re
from tempfile import mkdtemp
from datetime import datetime
from cement.core.controller import CementBaseController, expose
from cement.utils import fs, shell
from boss import VERSION
from boss.core import exc as boss_exc
    
if sys.version_info[0] < 3:
    from urllib2 import urlopen, HTTPError # pragma: no cover
else:
    from urllib.request import urlopen, HTTPError # pragma: no cover
    
    
BANNER = """

  _______  _______ _______ _______ 
 |   _   \|   _   |   _   |   _   |
 |.  1   /|.  |   |   1___|   1___|
 |.  _   \|.  |   |____   |____   |
 |:  1    |:  1   |:  1   |:  1   |
 |::.. .  |::.. . |::.. . |::.. . |
 `-------'`-------`-------`-------'
               (c) 2012 <NAME>
                           v%s
                           
""" % VERSION

ALLOWED_STR_METHODS = ['upper', 'lower', 'title', 'swapcase', 'strip']

class Template(object):
    def __init__(self, app, path):
        self.app = app
        self.basedir = fs.abspath(path)
        self.config = self._get_config()
        self._word_map = dict()
        self._vars = dict()
        
    def _get_config(self):
        config = None
        possibles = [
            fs.abspath(os.path.join(self.basedir, 'boss.json')),
            fs.abspath(os.path.join(self.basedir, 'boss.yml')),
            ]
        for path in possibles:
            if os.path.exists(path):
                if os.path.basename(path) == 'boss.json':
                    config = self._get_json_config(path)
                    break
                elif os.path.basename(path) == 'boss.yml':
                    config = self._get_yaml_config(path)
                    break
        
        if not config:
            raise boss_exc.BossTemplateError("No supported config found.")
                
        # fix it up with some defaults
        if not config.has_key('delimiter'):
            config['delimiter'] = '@'
        
        return config
        
    def _get_json_config(self, path):
        full_path = fs.abspath(path)
        if not os.path.exists(full_path):
            raise boss_exc.BossTemplateError("Invalid template config.")
        
        self.app.log.debug('loading template config %s' % full_path)
                    
        import json
        return json.load(open(full_path, 'r'))
        
    def _get_yaml_config(self, path):
        full_path = fs.abspath(path)
        if not os.path.exists(full_path):
            raise boss_exc.BossTemplateError("Invalid template config.")
        
        self.app.log.debug('loading template config %s' % full_path)
                    
        try:
            import yaml
        except ImportError as e:
            raise boss_exc.BossRuntimeError("Unable to import yaml.  " + 
                                            "Please install pyYaml.")
                
        return yaml.load(open(full_path, 'r'))
        
    def _populate_vars(self):
        if self.config.has_key('variables'):
            for var,question in self.config['variables'].items():
                if self.app.config.has_key('answers', var.lower()):
                    default = self.app.config.get('answers', var.lower())
                    res = raw_input("%s: [%s] " % (question, default))
                    if len(res) == 0:
                        res = default
                else:
                    res = raw_input("%s: " % question)
                self._vars[var] = res.strip()
                
    def _sub(self, txt):
        ### FIX ME: This needs serious refactoring... PLEASE
        
        # do per item substitution rather than entire txt to avoid variable
        # confusion.  Also allows us to call .capitalize, .lower, etc on the
        # string after substitution.
        for line in str(txt).split('\n'):
            for item in line.split(' '):
                # Not a template var? (generic pattern)
                pattern = "(.*)\%s(.*)\%s(.*)" % \
                    (self.config['delimiter'], self.config['delimiter'])
                    
                if not re.match(pattern, item):
                    continue
            
                for key,value in sorted(self._vars.items()):
                    if key in self._word_map:
                        continue
                    
                    pattern = "(.*)\%s(%s)\%s(.*)" % (
                        self.config['delimiter'], 
                        key, 
                        self.config['delimiter'],
                        )
                         
                    m = re.match(pattern, item)
                    if m:
                        map_key = "%s%s%s" % (
                            self.config['delimiter'], 
                            m.group(2),
                            self.config['delimiter'],
                            )
                        self._word_map[map_key] = str(value)
                    else:
                        pattern = "(.*)\%s(%s)\.([_a-z0-9]*)\%s(.*)" % (
                            self.config['delimiter'], 
                            key,
                            self.config['delimiter'],
                            )
                        m = re.match(pattern, item)
                        if m:
                            # string method calls?
                            if len(m.group(3)) > 0:
                                if m.group(3) in ALLOWED_STR_METHODS:
                                    fixed = str(getattr(value, m.group(3))())
                                else:
                                    self.app.log.debug("str method '%s' not allowed." % m.group(3))
                                    fixed = str(value)
                            else:
                                fixed = str(value)

                            new_key = "%s%s.%s%s" % (
                                self.config['delimiter'], 
                                m.group(2), 
                                m.group(3),
                                self.config['delimiter'],
                                )
                            self._word_map[new_key] = fixed
        
        # actually replace the text
        new_txt = txt
        for pattern,replacement in sorted(self._word_map.items()):
            new_txt = re.sub(pattern, replacement, new_txt)
            
        return new_txt
{'fraction_non_alphanumeric': 0.11432262231551751, 'fraction_numerical': 0.003956079444534151, 'mean_word_length': 1.1188847074923025, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5497181', 'n_tokens_mistral': 3415, 'n_tokens_neox': 3260, 'n_words': 720}