<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Staples is a simple static site generator based on the idea of processors
mapped to types of files. It includes a basic server for development, and
has no external requirements itself, beyond the Python Standard Library.
Specific processors, such as the included Django template renderer, will
have their own requirements.

Info and source: http://github.com/typeish/Staples

<NAME>, type(ish) inc
License: UNLICENSE
"""

__author__ = 'type(ish) inc'
__version__ = '0.1a'
__license__ = 'UNLICENSE'

import os, shutil, commands, glob, sys

# Default settings
PROJECT_ROOT = os.getcwd()
CONTENT_DIR = os.path.join(PROJECT_ROOT, 'content')
DEPLOY_DIR = os.path.join(PROJECT_ROOT, 'deploy')

REMOTE_ROOT = ''
DEPLOY_TRACKING_FILE = os.path.join(PROJECT_ROOT, 'staples_deploy_info.p')

INDEX_FILE = 'index.html'
IGNORE = ()
PROCESSORS = {}

# Look for a settings.py in the current working directory
sys.path.append(PROJECT_ROOT)
try:
    from settings import *
except ImportError:
    if __name__ == "__main__":
        print 'No settings.py found, using defaults.\n'
else:
    if __name__ == "__main__":
        print 'Found settings.py'

verbose=False

class File(object):
    
    def __init__(self, file_path, parent_ignored=False, **kwargs):
        self.source = file_path
        self.rel_path = file_path.replace(CONTENT_DIR,'').lstrip('/')
        self.rel_parent, self.name = os.path.split(self.source)
        self.ext = os.path.splitext(self.name)[1]
        self.parent_ignored = parent_ignored
    
    def process(self, **kwargs):
        if verbose:
            print 'Processing:', self.rel_path
        if self.is_directory:
            if '/' in PROCESSORS:
                p = PROCESSORS['/']
            else:
                p = handle_directory
        elif self.name in PROCESSORS:
            p = PROCESSORS[self.name]
        elif self.ext in PROCESSORS:
            p = PROCESSORS[self.ext]
        elif '*' in PROCESSORS:
            p = PROCESSORS['*']
        else:
            p = handle_others
        return p(self, **kwargs)
        
    @property
    def deploy_path(self): return os.path.join(DEPLOY_DIR, self.rel_path)
    
    @property
    def remote_path(self): return os.path.join(REMOTE_ROOT, self.rel_path)
    
    @property
    def mtime(self): return os.path.getmtime(self.source)
    
    @property
    def is_directory(self): return os.path.isdir(self.source)

    @property
    def ignorable(self):
        return ( self.name.startswith(".") or self.name.startswith("_")
                or self.name.endswith("~") or self.name in IGNORE )

# BUILD FUNCTIONS
###############################################################################
def build(**kwargs):
    print 'Starting build...\n'
    if verbose:
        print 'Removing any existing deploy directory'
    shutil.rmtree(DEPLOY_DIR, ignore_errors=True)
    
    if verbose:
        print 'Creating deploy directory: ', DEPLOY_DIR 
    os.mkdir(DEPLOY_DIR)

    if verbose:
        print 'Traversing content directory: %s...' % CONTENT_DIR

    build_directories(CONTENT_DIR, **kwargs)
    
    print '\nBuild done'

def build_directories(t_path, **kwargs):
    """
    Recursively traverses a given directory, calling the given file's handler.
    Keyword arguments are passed through to the handler.
    """
    for file in glob.glob( os.path.join(t_path, '*') ):
        File(file, **kwargs).process(**kwargs)

# DEFAULT HANDLERS
# These two functions basically just copy anything they are given over to
# the deploy directory.

def handle_directory(f, **kwargs):
    """
    Copies directories, unless they are ignorable or their
    parent was ignored.
    """
    if not f.ignorable and not f.parent_ignored:
        if verbose:
            print 'Making directory: ', f.deploy_path
        os.mkdir(f.deploy_path)
    else:
        if verbose:
            print 'Not duplicating directory'
        kwargs.update({'parent_ignored': True})
    build_directories(f.source, **kwargs)

def handle_others(f, **kwargs):
    """
    Simply copies files from the source path to the deploy path.
    """
    if not f.ignorable and not f.parent_ignored:
        if verbose:
            print 'Copying file to:', f.deploy_path
        commands.getoutput(u"cp %s %s" % (f.source, f.deploy_path))
    elif verbose:
        print 'Ignoring:', f.rel_path


# EXTRA HANDLERS

def handle_django(f, for_deployment=False, **kwargs):
    """
    Renders templates using the Django template rendering engine. If the
    template ends in .django, the resulting output filename has that removed.
    
    Requires:
        Django - any version that can handle the templates you give it
        settings.py - placed in the same directory and defines both
                      TEMPLATE_DIRS and CONTEXT
    """
    if not f.ignorable and not f.parent_ignored:
        from django.template.loader import render_to_string
        import settings
        os.environ['DJANGO_SETTINGS_MODULE'] = u"settings"
        deploy_path = f.deploy_path.replace('.django','')

        if verbose:
            print 'Rendering:', f.rel_path
    
        context = {}
        if settings.CONTEXT:
            context = settings.CONTEXT
        context['for_deployment'] = for_deployment
        rendered = render_to_string(f.source, context)

        if verbose:
            print 'Saving rendered output to:', deploy_path
        fout = open(deploy_path,'w')
        fout.write(rendered)
        fout.close()
    elif verbose:
        print 'Ignoring:', f.rel_path




# WATCH FUNCTIONS
###############################################################################

import datetime, time

def watch():
    """
    Initiates a full rebuild of the project, then watches for changed files.
    Once a second, it polls all of the files in the content directory.
    """
    watcher = DirWatcher(CONTENT_DIR)
    build()
    print 'Watching %s...' % CONTENT_DIR
    while True:
        time.sleep(1)
        changed = watcher.find_changed_files()
        if len(changed) > 0:
            for f in changed:
                f.process()

class DirWatcher(object):
    """
    Class that keeps track of the files in the content directory, and their<commit_msg>Notes about next handlers<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Staples is a simple static site generator based on the idea of processors
mapped to types of files. It includes a basic server for development, and
has no external requirements itself, beyond the Python Standard Library.
Specific processors, such as the included Django template renderer, will
have their own requirements.

Info and source: http://github.com/typeish/Staples

<NAME>, type(ish) inc
License: UNLICENSE
"""

__author__ = 'type(ish) inc'
__version__ = '0.1a'
__license__ = 'UNLICENSE'

import os, shutil, commands, glob, sys

# Default settings
PROJECT_ROOT = os.getcwd()
CONTENT_DIR = os.path.join(PROJECT_ROOT, 'content')
DEPLOY_DIR = os.path.join(PROJECT_ROOT, 'deploy')

REMOTE_ROOT = ''
DEPLOY_TRACKING_FILE = os.path.join(PROJECT_ROOT, 'staples_deploy_info.p')

INDEX_FILE = 'index.html'
IGNORE = ()
PROCESSORS = {}

# Look for a settings.py in the current working directory
sys.path.append(PROJECT_ROOT)
try:
    from settings import *
except ImportError:
    if __name__ == "__main__":
        print 'No settings.py found, using defaults.\n'
else:
    if __name__ == "__main__":
        print 'Found settings.py'

verbose=False

class File(object):
    
    def __init__(self, file_path, parent_ignored=False, **kwargs):
        self.source = file_path
        self.rel_path = file_path.replace(CONTENT_DIR,'').lstrip('/')
        self.rel_parent, self.name = os.path.split(self.source)
        self.ext = os.path.splitext(self.name)[1]
        self.parent_ignored = parent_ignored
    
    def process(self, **kwargs):
        if verbose:
            print 'Processing:', self.rel_path
        if self.is_directory:
            if '/' in PROCESSORS:
                p = PROCESSORS['/']
            else:
                p = handle_directory
        elif self.name in PROCESSORS:
            p = PROCESSORS[self.name]
        elif self.ext in PROCESSORS:
            p = PROCESSORS[self.ext]
        elif '*' in PROCESSORS:
            p = PROCESSORS['*']
        else:
            p = handle_others
        return p(self, **kwargs)
        
    @property
    def deploy_path(self): return os.path.join(DEPLOY_DIR, self.rel_path)
    
    @property
    def remote_path(self): return os.path.join(REMOTE_ROOT, self.rel_path)
    
    @property
    def mtime(self): return os.path.getmtime(self.source)
    
    @property
    def is_directory(self): return os.path.isdir(self.source)

    @property
    def ignorable(self):
        return ( self.name.startswith(".") or self.name.startswith("_")
                or self.name.endswith("~") or self.name in IGNORE )

# BUILD FUNCTIONS
###############################################################################
def build(**kwargs):
    print 'Starting build...\n'
    if verbose:
        print 'Removing any existing deploy directory'
    shutil.rmtree(DEPLOY_DIR, ignore_errors=True)
    
    if verbose:
        print 'Creating deploy directory: ', DEPLOY_DIR 
    os.mkdir(DEPLOY_DIR)

    if verbose:
        print 'Traversing content directory: %s...' % CONTENT_DIR

    build_directories(CONTENT_DIR, **kwargs)
    
    print '\nBuild done'

def build_directories(t_path, **kwargs):
    """
    Recursively traverses a given directory, calling the given file's handler.
    Keyword arguments are passed through to the handler.
    """
    for file in glob.glob( os.path.join(t_path, '*') ):
        File(file, **kwargs).process(**kwargs)

# DEFAULT HANDLERS
# These two functions basically just copy anything they are given over to
# the deploy directory.

def handle_directory(f, **kwargs):
    """
    Copies directories, unless they are ignorable or their
    parent was ignored.
    """
    if not f.ignorable and not f.parent_ignored:
        if verbose:
            print 'Making directory: ', f.deploy_path
        os.mkdir(f.deploy_path)
    else:
        if verbose:
            print 'Not duplicating directory'
        kwargs.update({'parent_ignored': True})
    build_directories(f.source, **kwargs)

def handle_others(f, **kwargs):
    """
    Simply copies files from the source path to the deploy path.
    """
    if not f.ignorable and not f.parent_ignored:
        if verbose:
            print 'Copying file to:', f.deploy_path
        commands.getoutput(u"cp %s %s" % (f.source, f.deploy_path))
    elif verbose:
        print 'Ignoring:', f.rel_path


# EXTRA HANDLERS

def handle_django(f, for_deployment=False, **kwargs):
    """
    Renders templates using the Django template rendering engine. If the
    template ends in .django, the resulting output filename has that removed.
    
    Requires:
        Django - any version that can handle the templates you give it
        settings.py - placed in the same directory and defines both
                      TEMPLATE_DIRS and CONTEXT
    """
    if not f.ignorable and not f.parent_ignored:
        from django.template.loader import render_to_string
        import settings
        os.environ['DJANGO_SETTINGS_MODULE'] = u"settings"
        deploy_path = f.deploy_path.replace('.django','')

        if verbose:
            print 'Rendering:', f.rel_path
    
        context = {}
        if settings.CONTEXT:
            context = settings.CONTEXT
        context['for_deployment'] = for_deployment
        rendered = render_to_string(f.source, context)

        if verbose:
            print 'Saving rendered output to:', deploy_path
        fout = open(deploy_path,'w')
        fout.write(rendered)
        fout.close()
    elif verbose:
        print 'Ignoring:', f.rel_path


def handle_gallery(f, **kwargs):
    pass
    # build a gallery using a folder of images, creating an index.html file
    # with all of the images, and an N.html file for each image with the full
    # sized image, up to CONTEXT.max_width

def handle_markdown(f, **kwargs):
    pass

# WATCH FUNCTIONS
###############################################################################

import datetime, time

def watch():
    """
    Initiates a full rebuild of the project, then watches for changed files.
    Once a second, it polls all of the files in the content directory.
    """
    watcher = DirWatcher(CONTENT_DIR)
    build()
    print 'Watching %s...' % CONTENT_DIR
    while True:
        time.sleep(1)
        changed = watcher.find_changed_files()
        if len(changed) > 0:
            for f in changed:
                f.process()

class DirWatcher(object):
    """
    Class that keeps track of the files in the content directory, and their
{'fraction_non_alphanumeric': 0.1219303699098539, 'fraction_numerical': 0.0009325458501709667, 'mean_word_length': 2.7344747533372025, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2422838', 'n_tokens_mistral': 3924, 'n_tokens_neox': 3764, 'n_words': 1132}