<commit_before>"""(c) All rights reserved. ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE, Switzerland, VPSI, 2017"""
import logging
import functools
import subprocess
import importlib
import sys
import io
import os
import csv
import string
import binascii
import random


def deprecated(message):
    """ This is a decorator which can be used to mark functions as deprecated. It will result in a
        warning being emmitted when the function is used.
        https://stackoverflow.com/questions/2536307/decorators-in-the-python-standard-lib-deprecated-specifically

        # Examples

        @deprecated
        def some_old_function(x,y):
            return x + y

        class SomeClass:
            @deprecated
            def some_old_method(self, x,y):
                return x + y
    """
    def decorator(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            print("WARNING: Call to deprecated function '{}'. {}".format(func.__name__, message))
            return func(*args, **kwargs)
        return new_func

    return decorator


class Utils(object):
    """Generic and all-purpose helpers"""

    @staticmethod
    def run_command(command, encoding=sys.stdout.encoding):
        """
        Execute the given command in a shell

        Argument keywords
        command -- command to execute
        encoding -- encoding to use

        Return
        False if error
        True if OK but no output from command
        Command output if there is one
        """
        try:
            # encode command properly for subprocess
            command_bytes = command.encode(encoding)
            # run command and log output
            proc = subprocess.run(command_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, shell=True)
            logging.debug("{} => {}".format(command, proc.stdout))
            # return output if got any, True otherwise
            if proc.stdout:
                # Second parameter "ignore" has been added because some plugins have 'strange' characters in their
                # name so 'decode' is failing and exits the script. Adding "ignore" as parameter prevent script from
                # exiting.
                text = proc.stdout.decode(encoding, "ignore")
                # get rid of final spaces, line return
                logging.debug(text.strip())
                return text.strip()
            return True

        except subprocess.CalledProcessError as err:
            # log error with content of stderr
            logging.error("command failed (code {}) with error <{}> => {}".format(
                          err.returncode,
                          err,
                          err.stderr))
            return False

    @classmethod
    def csv_stream_do_dict(cls, stream, delimiter=','):
        """
        Transform Stream (in CSV format) into a dictionnary

        Arguments keywords:
        stream -- stream containing infos to put in dictionnary
                  For stream information, have a look here: https://docs.python.org/3.5/library/io.html
        delimiter -- character to use to split infos coming from stream (CSV)

        Return: list of dictionnaries
        """
        rows = []
        reader = csv.DictReader(stream, delimiter=delimiter)
        for row in reader:
            rows.append(row)
        return rows

    @classmethod
    def csv_string_to_dict(cls, text, delimiter=','):
        """
        Transform a string (in CSV format) into a dictionnary

        Arguments keywords:
        text -- String containing CSV information
        delimiter -- character to use to split infos coming from string (CSV)
<commit_msg>Add necessary to ignore lines beginning with # in CSV file<commit_after>"""(c) All rights reserved. ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE, Switzerland, VPSI, 2017"""
import logging
import functools
import subprocess
import importlib
import sys
import io
import os
import csv
import string
import binascii
import random


def deprecated(message):
    """ This is a decorator which can be used to mark functions as deprecated. It will result in a
        warning being emmitted when the function is used.
        https://stackoverflow.com/questions/2536307/decorators-in-the-python-standard-lib-deprecated-specifically

        # Examples

        @deprecated
        def some_old_function(x,y):
            return x + y

        class SomeClass:
            @deprecated
            def some_old_method(self, x,y):
                return x + y
    """
    def decorator(func):
        @functools.wraps(func)
        def new_func(*args, **kwargs):
            print("WARNING: Call to deprecated function '{}'. {}".format(func.__name__, message))
            return func(*args, **kwargs)
        return new_func

    return decorator


class Utils(object):
    """Generic and all-purpose helpers"""

    @staticmethod
    def run_command(command, encoding=sys.stdout.encoding):
        """
        Execute the given command in a shell

        Argument keywords
        command -- command to execute
        encoding -- encoding to use

        Return
        False if error
        True if OK but no output from command
        Command output if there is one
        """
        try:
            # encode command properly for subprocess
            command_bytes = command.encode(encoding)
            # run command and log output
            proc = subprocess.run(command_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, shell=True)
            logging.debug("{} => {}".format(command, proc.stdout))
            # return output if got any, True otherwise
            if proc.stdout:
                # Second parameter "ignore" has been added because some plugins have 'strange' characters in their
                # name so 'decode' is failing and exits the script. Adding "ignore" as parameter prevent script from
                # exiting.
                text = proc.stdout.decode(encoding, "ignore")
                # get rid of final spaces, line return
                logging.debug(text.strip())
                return text.strip()
            return True

        except subprocess.CalledProcessError as err:
            # log error with content of stderr
            logging.error("command failed (code {}) with error <{}> => {}".format(
                          err.returncode,
                          err,
                          err.stderr))
            return False

    @classmethod
    def csv_stream_do_dict(cls, stream, delimiter=','):
        """
        Transform Stream (in CSV format) into a dictionnary

        Arguments keywords:
        stream -- stream containing infos to put in dictionnary
                  For stream information, have a look here: https://docs.python.org/3.5/library/io.html
        delimiter -- character to use to split infos coming from stream (CSV)

        Return: list of dictionnaries
        """
        rows = []
        # Getting stream content and ignoring lines beginning with # (treated as comment lines)
        reader = csv.DictReader(filter(lambda row: row[0] != '#', stream), delimiter=delimiter)
        for row in reader:
            rows.append(row)
        return rows

    @classmethod
    def csv_string_to_dict(cls, text, delimiter=','):
        """
        Transform a string (in CSV format) into a dictionnary

        Arguments keywords:
        text -- String containing CSV information
        delimiter -- character to use to split infos coming from string (CSV)

{'fraction_non_alphanumeric': 0.07971398305084745, 'fraction_numerical': 0.0035752118644067797, 'mean_word_length': 2.05542071197411, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2428237', 'n_tokens_mistral': 1963, 'n_tokens_neox': 1860, 'n_words': 686}