<commit_before>import os
import pwd
import grp
import errno
from shutil import copyfile
from quik import Template, FileLoader
import urllib
import autotest
from collections import namedtuple
from fileio.dirio import mkdirp
import shutil
from exthandlers import extension_handles
from log import log

def get_sq_ext(filename):
    """
    Gets the squadron extension from filename.
    Keyword arguments:
        filename -- the file to get the extension from
    """
    try:
        return filename[filename.rindex('~')+1:]
    except ValueError:
        return ''

def get_file_ext(filename):
    """
    Gets the extension (or compound extension) from a filename

    Keyword arguments:
        filename -- the file to get the extension from
    """
    root,ext = os.path.splitext(filename)
    if ext.lower() in ['.gz', '.bz2', '.xz']:
        other = os.path.splitext(root)[1]
        if other.lower() in ['.tar']:
            # Make sure we bundle file.tar.gz as 'tar.gz'
            return (other[1:] + ext).lower()

    return ext[1:].lower()


FileConfig = namedtuple('FileConfig', 'filepath atomic user group mode')
def parse_config(filename):
    """
    Parses a config.sq file which contains metadata about files in this
    directory.

    Keyword arguments:
        filename -- the config.sq file to open
    """
    # Conversion functions from string to the correct type, str is identity
    convert = {'atomic': bool, 'user':str, 'group':str, 'mode':str}
    require_dir = set(['atomic'])
    result = []
    with open(filename) as cfile:
        for line in cfile:
            # These are the default values
            item = {'atomic': False, 'user':None, 'group':None, 'mode':None}

            args = line.split()
            if len(args) < 2:
                raise ValueError('Line "{}" in file {} isn\'t formatted correctly'.format(line, filename))

            filepath = args[0]
            for arg in args[1:]:
                (key, value) = arg.split(':', 2)

                if key in require_dir:
                    # if this key requires us to be a directory, check
                    if not filepath.endswith(os.path.sep):
                        raise ValueError('Key {} requires entry {} to end with' +
                                ' a slash (must be directory) in file {}'.format(key, filepath, filename))

                #Only do work if we know about this parameter
                if key in convert:
                    item[key] = convert[key](value)
                else:
                    raise ValueError('Unknown config.sq value {} in file {}'.format(key, filename))
            result.append(FileConfig(filepath, item['atomic'], item['user'], item['group'], item['mode']))
    return result

def get_config(filename, config):
    path_items = filename.split(os.path.sep)
    accum = ""
    file_setting = FileConfig(filename, False, None, None, None)
    for item in path_items[:-1]:
        # loop over all the directories, but not the filename
        path = os.path.join(accum, item)
        if path + os.path.sep in config:
            file_setting = config[path + os.path.sep]

    if filename in config:
        file_setting = config[filename]

    return file_setting

def apply_config(filepath, file_config, dry_run):
    uid = -1
    gid = -1
    if file_config.user is not None:
        uid = pwd.getpwnam(file_config.user).pw_uid
    if file_config.group is not None:
        gid = grp.getgrnam(file_config.group).gr_gid

    if not dry_run:
        log.debug("Changing %s to uid %s gid %s", filepath, uid, gid)
        os.chown(filepath, uid, gid)

        if file_config.mode is not None:
            mode = int(file_config.mode, 8)
            log.debug("Changing mode of %s to %s", filepath, file_config.mode)
            os.chmod(filepath, mode)
    else:
        log.info("Would change %s to uid %s gid %s", filepath, uid, gid)

        if file_config.mode is not None:
            log.debug("Would change mode of %s to %s", filepath, file_config.mode)


class DirectoryRender:
    def __init__(self, basedir):
        if not os.path.isabs(basedir):
            basedir = os.path.abspath(basedir)
        self.loader = FileLoader(basedir)
        self.basedir = basedir

    def render(self, destdir, inputhash, dry_run = False,
            currpath = "", config = {}):
        """
        Transforms all templates and downloads all files in the directory
        supplied with the input values supplied. Output goes in destdir.

        Keyword arguments:
            destdir -- the directory to put the rendered files into
            inputhash -- the dictionary of input values
        """
        items = sorted(os.listdir(os.path.join(self.basedir, currpath)))<commit_msg>Only debug if there's a change<commit_after>import os
import pwd
import grp
import errno
from shutil import copyfile
from quik import Template, FileLoader
import urllib
import autotest
from collections import namedtuple
from fileio.dirio import mkdirp
import shutil
from exthandlers import extension_handles
from log import log

def get_sq_ext(filename):
    """
    Gets the squadron extension from filename.
    Keyword arguments:
        filename -- the file to get the extension from
    """
    try:
        return filename[filename.rindex('~')+1:]
    except ValueError:
        return ''

def get_file_ext(filename):
    """
    Gets the extension (or compound extension) from a filename

    Keyword arguments:
        filename -- the file to get the extension from
    """
    root,ext = os.path.splitext(filename)
    if ext.lower() in ['.gz', '.bz2', '.xz']:
        other = os.path.splitext(root)[1]
        if other.lower() in ['.tar']:
            # Make sure we bundle file.tar.gz as 'tar.gz'
            return (other[1:] + ext).lower()

    return ext[1:].lower()


FileConfig = namedtuple('FileConfig', 'filepath atomic user group mode')
def parse_config(filename):
    """
    Parses a config.sq file which contains metadata about files in this
    directory.

    Keyword arguments:
        filename -- the config.sq file to open
    """
    # Conversion functions from string to the correct type, str is identity
    convert = {'atomic': bool, 'user':str, 'group':str, 'mode':str}
    require_dir = set(['atomic'])
    result = []
    with open(filename) as cfile:
        for line in cfile:
            # These are the default values
            item = {'atomic': False, 'user':None, 'group':None, 'mode':None}

            args = line.split()
            if len(args) < 2:
                raise ValueError('Line "{}" in file {} isn\'t formatted correctly'.format(line, filename))

            filepath = args[0]
            for arg in args[1:]:
                (key, value) = arg.split(':', 2)

                if key in require_dir:
                    # if this key requires us to be a directory, check
                    if not filepath.endswith(os.path.sep):
                        raise ValueError('Key {} requires entry {} to end with' +
                                ' a slash (must be directory) in file {}'.format(key, filepath, filename))

                #Only do work if we know about this parameter
                if key in convert:
                    item[key] = convert[key](value)
                else:
                    raise ValueError('Unknown config.sq value {} in file {}'.format(key, filename))
            result.append(FileConfig(filepath, item['atomic'], item['user'], item['group'], item['mode']))
    return result

def get_config(filename, config):
    path_items = filename.split(os.path.sep)
    accum = ""
    file_setting = FileConfig(filename, False, None, None, None)
    for item in path_items[:-1]:
        # loop over all the directories, but not the filename
        path = os.path.join(accum, item)
        if path + os.path.sep in config:
            file_setting = config[path + os.path.sep]

    if filename in config:
        file_setting = config[filename]

    return file_setting

def apply_config(filepath, file_config, dry_run):
    uid = -1
    gid = -1
    if file_config.user is not None:
        uid = pwd.getpwnam(file_config.user).pw_uid
    if file_config.group is not None:
        gid = grp.getgrnam(file_config.group).gr_gid

    if not dry_run:
        if uid != -1 or gid != -1:
            log.debug("Changing %s to uid %s gid %s", filepath, uid, gid)
            os.chown(filepath, uid, gid)

        if file_config.mode is not None:
            mode = int(file_config.mode, 8)
            log.debug("Changing mode of %s to %s", filepath, file_config.mode)
            os.chmod(filepath, mode)
    else:
        if uid != -1 or gid != -1:
            log.info("Would change %s to uid %s gid %s", filepath, uid, gid)

        if file_config.mode is not None:
            log.debug("Would change mode of %s to %s", filepath, file_config.mode)


class DirectoryRender:
    def __init__(self, basedir):
        if not os.path.isabs(basedir):
            basedir = os.path.abspath(basedir)
        self.loader = FileLoader(basedir)
        self.basedir = basedir

    def render(self, destdir, inputhash, dry_run = False,
            currpath = "", config = {}):
        """
        Transforms all templates and downloads all files in the directory
        supplied with the input values supplied. Output goes in destdir.

        Keyword arguments:
            destdir -- the directory to put the rendered files into
            inputhash -- the dictionary of input values
        """
        items = sorted(os.listdir(os.path.join(self.basedir, currpath)))
{'fraction_non_alphanumeric': 0.10497237569060773, 'fraction_numerical': 0.003127280308558324, 'mean_word_length': 2.2544097693351426, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1835734', 'n_tokens_mistral': 2792, 'n_tokens_neox': 2710, 'n_words': 924}