<commit_before>#!/usr/bin/env python
# encoding: utf-8


import os
import re
import sys
import tempfile
try:
    from git import Repo
except ImportError:
    print 'GitPython module missing, please install it.'
    sys.exit()
try:
    from jinja2 import Environment, FileSystemLoader
except ImportError:
    print 'Jinja2 is required for tempalte processing, please install it.'
    sys.exit()
from shutil import copytree, move, rmtree, copy


class Template(object):

    is_git = False
    complete = False
    place_holders = {
        'PROJECT_NAME': 'project_name',
        'SETTINGS_DIR': 'template_settings_dir',
        'DB_NAME': 'db_name',
        'DB_USER': 'db_user',
        'DB_PASS': '<PASSWORD>',
        'DJANGO_SECRET_KEY': 'django_secret_key'}

    def __init__(self, config):

        self.config = config
        self.working_dir = os.popen('pwd').read().split()[0]
        self.set_project_root()
        self.set_template_variables()
        self.is_git()
        self.copy_template()

    def set_project_root(self):
        '''Set project root, based on working dir and project name.'''

        self.project_root = os.path.join(self.working_dir,
                                         self.config.project_name)

    def make_project_dir(self):
        '''Make the project director in current working directory.'''

        if not os.path.isdir(self.project_root):
            os.mkdir(self.project_root)
            if not os.path.isdir(self.project_root):
                self.config.cli_opts.error('Error creating project '
                                           'directory')
        else:
            self.config.cli_opts.error('%s already exists' % (
                self.project_root))

    def set_template_variables(self):
        ''' Replace self.place_holders defaults w/ config values. '''

        for place_holder in self.place_holders:
            config_value = getattr(self.config,
                    self.place_holders[place_holder], None)
            if config_value:
                self.place_holders[place_holder] = config_value

# TODO: git stuff should live in its own class
    def is_git(self):
        '''Detect if the user wants to use a git repository.'''

        if self.config.template.startswith('git+'):
            self.is_git = True
            self.git_repo_path = self.config.template.replace('git+', '')
            self.config.template = tempfile.mkdtemp(suffix='skeletor')
            print 'Using git to clone template from %s' % self.git_repo_path
            self.git_clone()

    def git_clone(self):
        '''Clone git repository into tmp directory.'''

        try:
            repo = Repo.init(self.config.template)
            repo.create_remote('origin', self.git_repo_path)
            origin = repo.remotes.origin
            origin.fetch()
            origin.pull('master')
        except:
            self.config.cli_opts.error('Error cloning repository')
        else:
            rmtree(os.path.join(self.config.template, '.git'))

    def copy_template(self):
        '''Moves template into current working dir.'''

        if os.path.isdir(self.config.template):
            self.make_project_dir()
            for file in os.listdir(self.config.template):
                path = os.path.join(self.config.template, file)
                if os.path.isdir(path):
                    copytree(path, os.path.join(self.project_root, file))
                else:
                    copy(path, self.project_root)
            self.swap_placeholders()
        else:
            self.config.cli_opts.error('Unable to copy template, directory '
                                       'does not exist')
        if self.is_git:
            rmtree(self.config.template)

    def rename(self, root, name):
        '''Rename a file or directory.'''

        e = re.compile(r'{{(.*?)}}')
        try:
            plain = e.findall(name)[0]
            if plain in self.place_holders:
                place_holder_val = self.place_holders[plain]
                origin = os.path.join(root, name)
                new_name = name.replace('{{%s}}' % plain, place_holder_val)
                new = os.path.join(root, new_name)
                move(origin, new)
        except IndexError:
            pass

    def rename_directories(self):
        '''Move directories with placeholder names.'''

        for root, dirs, files in os.walk(self.project_root):
            for d in dirs:
                self.rename(root, d)
        return False

    def rename_files(self):
        '''Move files with placeholder names.'''

        for root, dirs, files in os.walk(self.project_root):
            for f in files:
                self.rename(root, f)
        return False

    def swap_placeholders(self):
        '''Swap placeholders for real values.'''

        while self.rename_directories():
            continue

        while self.rename_files():
            continue

        for root, dirs, files in os.walk(self.project_root):
            jinja_tpl_loader = FileSystemLoader(root)
            jinja_env = Environment(loader=jinja_tpl_loader)
            for file in files:
                tpl = jinja_env.get_template(file)
                file_contents = tpl.render(self.place_holders)
                filepath = os.path.join(root, file)
                os.remove(filepath)
                with open(filepath, 'w') as f:
                    f.write(file_contents)<commit_msg>Recursive dir / tpl renaming + excluding .git and .hg directories<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import sys
import tempfile
try:
    from git import Repo
except ImportError:
    print 'GitPython module missing, please install it.'
    sys.exit()
try:
    from jinja2 import Environment, FileSystemLoader
except ImportError:
    print 'Jinja2 is required for tempalte processing, please install it.'
    sys.exit()
from shutil import copytree, move, rmtree, copy


class Template(object):

    is_git = False
    complete = False
    place_holders = {
        'PROJECT_NAME': 'project_name',
        'SETTINGS_DIR': 'template_settings_dir',
        'DB_NAME': 'db_name',
        'DB_USER': 'db_user',
        'DB_PASS': '<PASSWORD>',
        'DJANGO_SECRET_KEY': 'django_secret_key'}
    exclude_dirs = ['.git', '.hg']
    directory_map = {}

    def __init__(self, config):

        self.config = config
        self.working_dir = os.popen('pwd').read().split()[0]
        self.set_project_root()
        self.set_template_variables()
        self.is_git()
        self.copy_template()

    def set_project_root(self):
        '''Set project root, based on working dir and project name.'''

        self.project_root = os.path.join(self.working_dir,
                                         self.config.project_name)

    def make_project_dir(self):
        '''Make the project director in current working directory.'''

        if not os.path.isdir(self.project_root):
            os.mkdir(self.project_root)
            if not os.path.isdir(self.project_root):
                self.config.cli_opts.error('Error creating project '
                                           'directory')
        else:
            self.config.cli_opts.error('%s already exists' % (
                self.project_root))

    def set_template_variables(self):
        ''' Replace self.place_holders defaults w/ config values. '''

        for place_holder in self.place_holders:
            config_value = getattr(self.config,
                    self.place_holders[place_holder], None)
            if config_value:
                self.place_holders[place_holder] = config_value

# TODO: git stuff should live in its own class
    def is_git(self):
        '''Detect if the user wants to use a git repository.'''

        if self.config.template.startswith('git+'):
            self.is_git = True
            self.git_repo_path = self.config.template.replace('git+', '')
            self.config.template = tempfile.mkdtemp(suffix='skeletor')
            print 'Using git to clone template from %s' % self.git_repo_path
            self.git_clone()

    def git_clone(self):
        '''Clone git repository into tmp directory.'''

        try:
            repo = Repo.init(self.config.template)
            repo.create_remote('origin', self.git_repo_path)
            origin = repo.remotes.origin
            origin.fetch()
            origin.pull('master')
        except:
            self.config.cli_opts.error('Error cloning repository')
        else:
            rmtree(os.path.join(self.config.template, '.git'))

    def copy_template(self):
        '''Moves template into current working dir.'''

        if os.path.isdir(self.config.template):
            self.make_project_dir()
            for file in os.listdir(self.config.template):
                path = os.path.join(self.config.template, file)
                if os.path.isdir(path):
                    copytree(path, os.path.join(self.project_root, file))
                else:
                    copy(path, self.project_root)
            self.swap_placeholders()
        else:
            self.config.cli_opts.error('Unable to copy template, directory '
                                       'does not exist')
        if self.is_git:
            rmtree(self.config.template)

    def rename(self, root, name):
        '''Rename a file or directory.'''

        e = re.compile(r'__(.*?)__')
        try:
            plain = e.findall(name)[0]
            if plain in self.place_holders:
                place_holder_val = self.place_holders[plain]
                origin = os.path.join(root, name)
                new_name = name.replace('__%s__' % plain, place_holder_val)
                new = os.path.join(root, new_name)
                move(origin, new)
                return True
            else:
                return False
        except IndexError:
            pass

    def rename_directories(self):
        '''Move directories with placeholder names.'''

        for root, dirs, files in os.walk(self.project_root):
            for d in dirs:
                filepath = os.path.join(root, d)
                dirs = filepath.split('/')
                exclude = False
                for dir in dirs:
                    if dir in self.exclude_dirs:
                        exclude = True
                if not exclude:
                    if self.rename(root, d):
                        self.rename_directories()
        return False

    def rename_files(self):
        '''Move files with placeholder names.'''

        for root, dirs, files in os.walk(self.project_root):
            for f in files:
                filepath = os.path.join(root, f)
                dirs = filepath.split('/')
                exclude = False
                for dir in dirs:
                    if dir in self.exclude_dirs:
                        exclude = True
                if not exclude:
                    if self.rename(root, f):
                        self.rename_files()
        return False

    def swap_placeholders(self):
        '''Swap placeholders for real values.'''

        while self.rename_directories():
            continue

        while self.rename_files():
            continue

        for root, dirs, files in os.walk(self.project_root):
            jinja_tpl_loader = FileSystemLoader(root)
            jinja_env = Environment(loader=jinja_tpl_loader)
            for file in files:
                filepath = os.path.join(root, file)
                exclude = False
                dirs = filepath.split('/')
                for dir in dirs:
                    if dir in self.exclude_dirs:
                        exclude = True
                if not exclude:
                    tpl = jinja_env.get_template(file)
                    file_contents = tpl.render(self.place_holders)
                    os.remove(filepath)
                    with open(filepath, 'w') as f:
                        f.write(file_contents)
{'fraction_non_alphanumeric': 0.10003331667499583, 'fraction_numerical': 0.0008329168748958854, 'mean_word_length': 1.7716989843028623, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4852342', 'n_tokens_mistral': 3399, 'n_tokens_neox': 3250, 'n_words': 805}