<commit_before># -*- coding: utf-8 -*-

# <NAME>
# Summer 2014
# Univerity of Colorado

import os
import shutil
import copy
import subprocess
import logging

import config

import backend_redis as backend
import structs
import env

_TST_DIR = "test"
_SUB_DIR = "submission"
_FILE_SANITIZERS = ["dos2unix", "mac2unix"]

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.NullHandler())


class Env(env.Env):

    def __init__(self, asn, sub, tst, run):

        # Call Parent
        super(Env, self).__init__(asn, sub, tst, run)
        msg = "envmod_local: Initializing environment"
        logger.info(self._format_msg(msg))

        # Setup Factory
        FileFactory = backend.UUIDFactory(structs.File)

        # Get Files
        sub_file_uuids = sub.list_files()
        sub_files = [FileFactory.from_existing(file_uuid) for file_uuid in sub_file_uuids]
        tst_file_uuids = tst.list_files()
        tst_files = [FileFactory.from_existing(file_uuid) for file_uuid in tst_file_uuids]

        # Setup Directories
        self.wd = os.path.abspath("{:s}/{:s}/".format(config.ENV_LOCAL_TMP_PATH, str(run).lower()))
        self.wd_tst = "{:s}/{:s}".format(self.wd, _TST_DIR)
        self.wd_sub = "{:s}/{:s}".format(self.wd, _SUB_DIR)
        os.makedirs(self.wd)
        os.makedirs(self.wd_tst)
        os.makedirs(self.wd_sub)
        os.chmod(self.wd_sub, 0777)

        # Copy Tester Files
        self.tst_files = []
        for fle in tst_files:
            fle_cpy = self.copy_fle(fle.get_dict(), self.wd_tst)
            self.tst_files.append(fle_cpy)

        # Copy Submission Files
        self.sub_files = []
        for fle in sub_files:
            fle_cpy = self.copy_fle(fle.get_dict(), self.wd_sub)
            self.sub_files.append(fle_cpy)

        # Sanitze Files
        for fle in (self.tst_files + self.sub_files):
            for pgm in _FILE_SANITIZERS:
                cmd_sanitize = [pgm, fle['path']]
                p = subprocess.Popen(cmd_sanitize, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdout, stderr = p.communicate()
                ret = p.returncode
                if stdout:
                    msg = "envmod_local: '{:s}' stdout: {:s}".format(pgm, stdout)
                    logger.debug(self._format_msg(msg))
                if stderr:
                    msg = "envmod_local: '{:s}' stderr: {:s}".format(pgm, stderr)
                    logger.debug(self._format_msg(msg))
                if (ret != 0):
                    msg = "envmod_local: "
                    msg += "Sanitize pgm '{:s}' failed ".format(pgm)
                    msg += "on file '{:s}': ".format(fle_cpy['name'])
                    msg += "{:s}".format(stderr)
                    logger.warning(self._format_msg(msg))

        # Clean and Setup Env Vars
        self._env_vars = {}
        for var in os.environ:
            if not var.startswith("COGS"):
                self._env_vars[var] = os.environ[var]
        self._env_vars['LANG'] = "C.UTF-8"
        msg = "env_vars: {:s}".format(self._env_vars)
        logger.info(self._format_msg(msg))

    def copy_fle(self, fle, dst_dir):

        new_fle = copy.copy(fle)

        # Setup Paths<commit_msg>Removed APACHE envvars from env_local<commit_after># -*- coding: utf-8 -*-

# <NAME>
# Summer 2014
# Univerity of Colorado

import os
import shutil
import copy
import subprocess
import logging

import config

import backend_redis as backend
import structs
import env

_TST_DIR = "test"
_SUB_DIR = "submission"
_FILE_SANITIZERS = ["dos2unix", "mac2unix"]

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.NullHandler())


class Env(env.Env):

    def __init__(self, asn, sub, tst, run):

        # Call Parent
        super(Env, self).__init__(asn, sub, tst, run)
        msg = "envmod_local: Initializing environment"
        logger.info(self._format_msg(msg))

        # Setup Factory
        FileFactory = backend.UUIDFactory(structs.File)

        # Get Files
        sub_file_uuids = sub.list_files()
        sub_files = [FileFactory.from_existing(file_uuid) for file_uuid in sub_file_uuids]
        tst_file_uuids = tst.list_files()
        tst_files = [FileFactory.from_existing(file_uuid) for file_uuid in tst_file_uuids]

        # Setup Directories
        self.wd = os.path.abspath("{:s}/{:s}/".format(config.ENV_LOCAL_TMP_PATH, str(run).lower()))
        self.wd_tst = "{:s}/{:s}".format(self.wd, _TST_DIR)
        self.wd_sub = "{:s}/{:s}".format(self.wd, _SUB_DIR)
        os.makedirs(self.wd)
        os.makedirs(self.wd_tst)
        os.makedirs(self.wd_sub)
        os.chmod(self.wd_sub, 0777)

        # Copy Tester Files
        self.tst_files = []
        for fle in tst_files:
            fle_cpy = self.copy_fle(fle.get_dict(), self.wd_tst)
            self.tst_files.append(fle_cpy)

        # Copy Submission Files
        self.sub_files = []
        for fle in sub_files:
            fle_cpy = self.copy_fle(fle.get_dict(), self.wd_sub)
            self.sub_files.append(fle_cpy)

        # Sanitze Files
        for fle in (self.tst_files + self.sub_files):
            for pgm in _FILE_SANITIZERS:
                cmd_sanitize = [pgm, fle['path']]
                p = subprocess.Popen(cmd_sanitize, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdout, stderr = p.communicate()
                ret = p.returncode
                if stdout:
                    msg = "envmod_local: '{:s}' stdout: {:s}".format(pgm, stdout)
                    logger.debug(self._format_msg(msg))
                if stderr:
                    msg = "envmod_local: '{:s}' stderr: {:s}".format(pgm, stderr)
                    logger.debug(self._format_msg(msg))
                if (ret != 0):
                    msg = "envmod_local: "
                    msg += "Sanitize pgm '{:s}' failed ".format(pgm)
                    msg += "on file '{:s}': ".format(fle_cpy['name'])
                    msg += "{:s}".format(stderr)
                    logger.warning(self._format_msg(msg))

        # Clean and Setup Env Vars
        self._env_vars = {}
        for var in os.environ:
            if var.startswith("COGS"):
                pass
            elif var.startswith("APACHE"):
                pass
            else:
                self._env_vars[var] = os.environ[var]
        self._env_vars['LANG'] = "C.UTF-8"
        msg = "env_vars: {:s}".format(self._env_vars)
        logger.info(self._format_msg(msg))

    def copy_fle(self, fle, dst_dir):

        new_fle = copy.copy(fle)

        # Setup Paths
{'fraction_non_alphanumeric': 0.12441562358618609, 'fraction_numerical': 0.003920977228170713, 'mean_word_length': 2.0990654205607475, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63183', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2201, 'n_words': 429}