<commit_before>#   Copyright 2017 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import ruamel.yaml as yaml
import markdown
from mako.lookup import TemplateLookup
from mako.template import Template
import sys, argparse, fnmatch, os
import copy, shutil
from . import create_default_files


DEFAULT_SITE_CONFIG = 'site.yaml'
SITE_DIR_SEPARATOR = '/'
TEMPLATE_EXTENTION = '.mako'
VERBOSE = True

class MySite(object):
    def __init__(self):
        self.config = dict()
        self.pages = dict()
        self.folders = dict()
def file_type(filename):
    """
    Detects the file type based on file name for filtering
    :param filename: Filename to be detected
    :return: type
    """
    name, ext = os.path.splitext(filename)
    if ext.lower() == '.md':
        return "markdown"
    elif ext.lower() == '.htm':
        return "html"
    elif ext.lower() == '.html':
        return "html"
    return ""


def new_site(sitepath, sitetemplate=None):
    """
    Create a new Edward site
    :param sitepath: Path where the new Edward site is going to be created
    :return: True if all went well
    """
    create_default_files.create_folders(sitepath)
    create_default_files.create_conf(os.path.join(sitepath, DEFAULT_SITE_CONFIG))
    if sitetemplate:
        # copy files from the template
        create_default_files.copy_files(sitepath, sitetemplate)
    return True

def render_site(sitepath, outpath=None):
    """
    Render the files in the Edward site directory
    :param sitepath: Path to Edward site directory
    :param outpath: Path to render the site to
    :return: True if all went well
    """
    # load site.yaml
    site = MySite()
    with open(os.path.join(sitepath, DEFAULT_SITE_CONFIG)) as infile:
        site.config = yaml.safe_load(infile)
        infile.close()
    site.pages = dict()
    # get root folders of site
    root_folders = os.scandir(sitepath)
    for folder in root_folders:
        if folder.is_dir():
            if not fnmatch.fnmatch(folder.name, site.config['exclude']):
                site.folders[folder.name] = folder.name
    # scan all templates
    template_dict = dict()
    ts = os.scandir(os.path.join(sitepath, '_templates'))
    for t in ts:
        if t.is_file():
            template_dict[os.path.splitext(t.name)[0]]=t.name
    if VERBOSE:
        print("The following templates have been found:", template_dict)
    #print(template_dict)
    # create render directory
    if not outpath:
        raise "please state an output directory"
    if not os.path.exists(outpath):
        os.mkdir(outpath)
    # go through the whole site, generating our site construct holding all information of the files within
    for dirpath, dirnames, filenames in os.walk(sitepath):
        if os.path.abspath(outpath) in os.path.abspath(dirpath):
            continue
        #print("path: " , os.path.abspath(dirpath), os.path.abspath(outpath))
        # match all files in the "interpret" list of site.yaml
        matched_files = []
        for pat in site.config['interpret']:
            matched_files += fnmatch.filter(filenames,pat)
        #print(matched_files)
        # read in frontmatter of all matched files
        for filename in matched_files:
            # remove filename from filenames, because will will simply copy all other files
            filenames.remove(filename)
            # now read in the file
            with open(os.path.join(dirpath, filename)) as infile:
                front_matter, content = list(yaml.safe_load_all(infile))[:2]
                infile.close()
            #print(front_matter)
            #print(content)
            front_matter["filepath"] = os.path.join(dirpath, filename)
            # get filetype - mostly markdown or html right now
            front_matter["filetype"] = file_type(filename)
            # we use the permalink as page ID
            if not 'permalink' in front_matter:
                # set permalink based on filename and relative path
                # we assume that os.walk always uses "/" as path separator. On the mac it does.
                htmlpath = dirpath.split(sitepath)[1] + SITE_DIR_SEPARATOR
                # remove leading separator
                if htmlpath[:len(SITE_DIR_SEPARATOR)] == SITE_DIR_SEPARATOR:
                    htmlpath = htmlpath[len(SITE_DIR_SEPARATOR):]
                front_matter['permalink'] = htmlpath + os.path.splitext(filename)[0] + site.config['html extention']
            # calculate relative path to root folder
            relpath = "../" * (dirpath.split(sitepath)[1].count("/"))
            #print(relpath)
            # add folder info
            front_matter['folders'] = dict()
            for folder in site.folders:
                front_matter['folders'][folder] = relpath + site.folders[folder]
            # now copy all that info to the pages dict in the site object
            site.pages[front_matter['permalink']]=copy.deepcopy(front_matter)
            #print(front_matter)
        # now all files left in filenames are files we will simply copy
        # unless we are in an ignored directory
        if fnmatch.fnmatch(os.path.split(dirpath)[1], site.config['exclude']):
            continue
        # calculate target directory
        targetdir = os.path.join(outpath, dirpath.split(sitepath)[1][1:])
        #print(targetdir)
        for fname in filenames:
            if fname.lower() == "site.yaml":
                continue
            # copy file
            #print("copy file:", os.path.join(dirpath, fname), os.path.join(targetdir, fname))
            shutil.copyfile(os.path.join(dirpath, fname), os.path.join(targetdir, fname))
        for dname in dirnames:
            if not os.path.abspath(outpath) in os.path.abspath(os.path.join(dirpath, dname)):
                if not fnmatch.fnmatch(dname, site.config['exclude']):
                    #make directory<commit_msg>added exeption for mako and site.pages['basepath'] that gives the path to the base of the site relative to the page.<commit_after>#   Copyright 2017 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import ruamel.yaml as yaml
import markdown
from mako.lookup import TemplateLookup
from mako.template import Template
import sys, argparse, fnmatch, os
import copy, shutil
from . import create_default_files
from mako import exceptions

DEFAULT_SITE_CONFIG = 'site.yaml'
SITE_DIR_SEPARATOR = '/'
TEMPLATE_EXTENTION = '.mako'
VERBOSE = True

class MySite(object):
    def __init__(self):
        self.config = dict()
        self.pages = dict()
        self.folders = dict()
def file_type(filename):
    """
    Detects the file type based on file name for filtering
    :param filename: Filename to be detected
    :return: type
    """
    name, ext = os.path.splitext(filename)
    if ext.lower() == '.md':
        return "markdown"
    elif ext.lower() == '.htm':
        return "html"
    elif ext.lower() == '.html':
        return "html"
    return ""


def new_site(sitepath, sitetemplate=None):
    """
    Create a new Edward site. If the sitetemplate parameter is empty, we will only create
    the usual directories and a DEFAULT_SITE_CONFIG file
    :param sitepath: Path where the new Edward site is going to be created
    :return: True if all went well
    """
    create_default_files.create_folders(sitepath)
    create_default_files.create_conf(os.path.join(sitepath, DEFAULT_SITE_CONFIG))
    if sitetemplate:
        # copy files from the template
        create_default_files.copy_files(sitepath, sitetemplate)
    return True

def render_site(sitepath, outpath=None):
    """
    Render the files in the Edward site directory
    :param sitepath: Path to Edward site directory
    :param outpath: Path to render the site to
    :return: True if all went well
    """
    # load site.yaml
    site = MySite()
    with open(os.path.join(sitepath, DEFAULT_SITE_CONFIG)) as infile:
        site.config = yaml.safe_load(infile)
        infile.close()
    site.pages = dict()
    # get root folders of site
    root_folders = os.scandir(sitepath)
    for folder in root_folders:
        if folder.is_dir():
            if not fnmatch.fnmatch(folder.name, site.config['exclude']):
                site.folders[folder.name] = folder.name
    # scan all templates
    template_dict = dict()
    ts = os.scandir(os.path.join(sitepath, '_templates'))
    for t in ts:
        if t.is_file():
            template_dict[os.path.splitext(t.name)[0]]=t.name
    if VERBOSE:
        print("The following templates have been found:", template_dict)
    #print(template_dict)
    # create render directory
    if not outpath:
        raise "please state an output directory"
    if not os.path.exists(outpath):
        os.mkdir(outpath)
    # go through the whole site, generating our site construct holding all information of the files within
    for dirpath, dirnames, filenames in os.walk(sitepath):
        if os.path.abspath(outpath) in os.path.abspath(dirpath):
            continue
        #print("path: " , os.path.abspath(dirpath), os.path.abspath(outpath))
        # match all files in the "interpret" list of site.yaml
        matched_files = []
        for pat in site.config['interpret']:
            matched_files += fnmatch.filter(filenames,pat)
        #print(matched_files)
        # read in frontmatter of all matched files
        for filename in matched_files:
            # remove filename from filenames, because will will simply copy all other files
            filenames.remove(filename)
            # now read in the file
            with open(os.path.join(dirpath, filename)) as infile:
                front_matter, content = list(yaml.safe_load_all(infile))[:2]
                infile.close()
            #print(front_matter)
            #print(content)
            front_matter["filepath"] = os.path.join(dirpath, filename)
            # get filetype - mostly markdown or html right now
            front_matter["filetype"] = file_type(filename)
            # we use the permalink as page ID
            if not 'permalink' in front_matter:
                # set permalink based on filename and relative path
                # we assume that os.walk always uses "/" as path separator. On the mac it does.
                htmlpath = dirpath.split(sitepath)[1] + SITE_DIR_SEPARATOR
                # remove leading separator
                if htmlpath[:len(SITE_DIR_SEPARATOR)] == SITE_DIR_SEPARATOR:
                    htmlpath = htmlpath[len(SITE_DIR_SEPARATOR):]
                front_matter['permalink'] = htmlpath + os.path.splitext(filename)[0] + site.config['html extention']
            # calculate relative path to root folder
            relpath = "../" * (dirpath.split(sitepath)[1].count("/"))
            site.pages['basepath'] = relpath
            #print(relpath)
            # add folder info
            front_matter['folders'] = dict()
            for folder in site.folders:
                front_matter['folders'][folder] = relpath + site.folders[folder]
            # now copy all that info to the pages dict in the site object
            site.pages[front_matter['permalink']]=copy.deepcopy(front_matter)
            #print(front_matter)
        # now all files left in filenames are files we will simply copy
        # unless we are in an ignored directory
        if fnmatch.fnmatch(os.path.split(dirpath)[1], site.config['exclude']):
            continue
        # calculate target directory
        targetdir = os.path.join(outpath, dirpath.split(sitepath)[1][1:])
        #print(targetdir)
        for fname in filenames:
            if fname.lower() == "site.yaml":
                continue
            # copy file
            #print("copy file:", os.path.join(dirpath, fname), os.path.join(targetdir, fname))
            shutil.copyfile(os.path.join(dirpath, fname), os.path.join(targetdir, fname))
        for dname in dirnames:
            if not os.path.abspath(outpath) in os.path.abspath(os.path.join(dirpath, dname)):
                if not fnmatch.fnmatch(dname, site.config['exclude']):
                    #make directory
{'fraction_non_alphanumeric': 0.09302325581395349, 'fraction_numerical': 0.001944215957526359, 'mean_word_length': 2.5465393794749405, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7611744', 'n_tokens_mistral': 3881, 'n_tokens_neox': 3823, 'n_words': 1213}