<commit_before>import json
import logging
import socket
import subprocess
import psutil
import time

GSTD_PROCNAME = 'gstd'
terminator = '\x00'.encode('utf-8')
max_size = 8192
def recvall(sock):
    buf = b''
    count = max_size
    try:
        while count:
            newbuf = sock.recv(max_size//8)
            if not newbuf: return None
            if terminator in newbuf:
                # this is the last item
                buf += newbuf[:newbuf.find(terminator)]
                break
            else:
                buf += newbuf
                count -= len(newbuf)
    except socket.error:
        buf = json.dumps({"error":"socket error", "msg": traceback.format_exc(), "code": -1 })
    return buf

# Add color to logging output
COLORS = {
    'WARNING': '33m',
    'INFO': '37m',
    'DEBUG': '34m',
    'CRITICAL': '35m',
    'ERROR': '31m'
}
class colorFormatter(logging.Formatter):
    def __init__(self, msg):
        logging.Formatter.__init__(self, msg)

    def format(self, record):
        if record.levelname in COLORS:
            record.levelname = "\033[1;" + COLORS[record.levelname] + record.levelname + "\033[0m"
        return logging.Formatter.format(self, record)

class client(object):
    def __init__(self, ip='localhost', port=5000, logfile=None, loglevel='ERROR'):
        
        # Init the logger
        self.logger = logging.getLogger('GSTD')
        self.logger.setLevel(logging.DEBUG)
        # Select to log in a file or console
        if logfile:
            # log in file
            log = logging.FileHandler(logfile)
        else:
            # log in console
            log = logging.StreamHandler()
        # Set log format with colors
        log.setFormatter(colorFormatter("%(asctime)22s  %(levelname)s    \t%(message)s"))
        # Set log level
        numeric_level = getattr(logging, loglevel.upper(), None)
        if isinstance(numeric_level, int):
            log.setLevel(numeric_level)
        else:
            log.setLevel(logging.ERROR)
        self.logger.addHandler(log)

        self.ip = ip
        self.port = port
        self.proc = None
        self.pipes = []
        self.gstd_started = False
        self.logger.info('Starting GSTD instance with ip=%s port=%d logfile=%s loglevel=%s', self.ip, self.port, logfile, loglevel)
        self.test_gstd()

    def __del__(self):
        self.logger.info('Destroying GSTD instance with ip=%s port=%d', self.ip, self.port)
        self.logger.info('Deleting pipelines...')
        while (self.pipes != []):
            ret = self.pipeline_delete(self.pipes[0])
            if (ret != 0):
                self.pipes.pop(0)
        if (self.gstd_started):
            self.logger.info('Killing GStreamer Daemon process...')
            self.proc.kill()

    def socket_send(self, line):
        self.logger.debug('GSTD socket sending line: %s', line)
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.connect((self.ip, self.port))
            s.send(' '.join(line).encode('utf-8'))
            data = recvall(s)
            data = data.decode('utf-8')
        except socket.error:
            self.logger.error('GSTD socket error')
            data = None
        self.logger.debug('GSTD socket received answer:\n %s', data)
        return data

    def start_gstd(self):
        try:
            gstd_bin = subprocess.check_output(['which',GSTD_PROCNAME])
            gstd_bin = gstd_bin.rstrip()
            self.logger.info('Startting GStreamer Daemon...')
            subprocess.Popen([gstd_bin])
            time.sleep(3)
            if self.test_gstd():
                self.gstd_started = True
                self.logger.info("GStreamer Daemon started successfully!")
                return True
            else:
                self.logger.info("GStreamer Daemon did not start correctly...")
                return False
        except subprocess.CalledProcessError:
            # Did not find gstd
            self.logger.error("GStreamer Daemon is not running and it is not installed.")
            self.logger.error("To get GStreamer Daemon, visit https://www.ridgerun.com/gstd.")
        return False

    def test_gstd(self):
        if self.ip not in ['localhost', '127.0.0.1']:
            # bypass process check, we don't know how to start gstd remotely<commit_msg>Improve logger to log each message only once<commit_after>import json
import logging
import socket
import subprocess
import psutil
import time
import traceback
import sys

GSTD_PROCNAME = 'gstd'
terminator = '\x00'.encode('utf-8')
max_size = 8192
def recvall(sock):
    buf = b''
    count = max_size
    try:
        while count:
            newbuf = sock.recv(max_size//8)
            if not newbuf: return None
            if terminator in newbuf:
                # this is the last item
                buf += newbuf[:newbuf.find(terminator)]
                break
            else:
                buf += newbuf
                count -= len(newbuf)
    except socket.error:
        buf = json.dumps({"error":"socket error", "msg": traceback.format_exc(), "code": -1 })
    return buf

# Add color to logging output
COLORS = {
    'WARNING': '33m',
    'INFO': '37m',
    'DEBUG': '34m',
    'CRITICAL': '35m',
    'ERROR': '31m'
}
class colorFormatter(logging.Formatter):
    def __init__(self, msg):
        logging.Formatter.__init__(self, msg)

    def format(self, record):
        if record.levelname in COLORS:
            record.levelname = "\033[1;" + COLORS[record.levelname] + record.levelname + "\033[0m"
        return logging.Formatter.format(self, record)

class client(object):
    def __init__(self, ip='localhost', port=5000, logfile=None, loglevel='ERROR'):
        
        # Init the logger
        self.logger = logging.getLogger('GSTD')
        self.logger.setLevel(logging.DEBUG)
        # Select to log in a file or console
        if logfile:
            # log in file
            self.log = logging.FileHandler(logfile)
        else:
            # log in console
            self.log = logging.StreamHandler()
        # Set log format with colors
        self.log.setFormatter(colorFormatter("%(asctime)22s  %(levelname)s    \t%(message)s"))
        # Set log level
        numeric_level = getattr(logging, loglevel.upper(), None)
        if isinstance(numeric_level, int):
            self.log.setLevel(numeric_level)
        else:
            self.log.setLevel(logging.ERROR)
        self.logger.addHandler(self.log)

        self.ip = ip
        self.port = port
        self.proc = None
        self.pipes = []
        self.gstd_started = False
        self.logger.info('Starting GSTD instance with ip=%s port=%d logfile=%s loglevel=%s', self.ip, self.port, logfile, loglevel)
        self.test_gstd()

    def __del__(self):
        self.logger.info('Destroying GSTD instance with ip=%s port=%d', self.ip, self.port)
        self.logger.info('Deleting pipelines...')
        while (self.pipes != []):
            ret = self.pipeline_delete(self.pipes[0])
            if (ret != 0):
                self.pipes.pop(0)
        if (self.gstd_started):
            self.logger.info('Killing GStreamer Daemon process...')
            self.proc.kill()
        self.logger.removeHandler(self.log)

    def socket_send(self, line):
        self.logger.debug('GSTD socket sending line: %s', line)
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.connect((self.ip, self.port))
            s.send(' '.join(line).encode('utf-8'))
            data = recvall(s)
            data = data.decode('utf-8')
            s.close()
        except socket.error:
            s.close()
            self.logger.error('GSTD socket error')
            data = None
        self.logger.debug('GSTD socket received answer:\n %s', data)
        return data

    def start_gstd(self):
        try:
            gstd_bin = subprocess.check_output(['which',GSTD_PROCNAME])
            gstd_bin = gstd_bin.rstrip()
            self.logger.info('Startting GStreamer Daemon...')
            subprocess.Popen([gstd_bin])
            time.sleep(3)
            if self.test_gstd():
                self.gstd_started = True
                self.logger.info("GStreamer Daemon started successfully!")
                return True
            else:
                self.logger.info("GStreamer Daemon did not start correctly...")
                return False
        except subprocess.CalledProcessError:
            # Did not find gstd
            self.logger.error("GStreamer Daemon is not running and it is not installed.")
            self.logger.error("To get GStreamer Daemon, visit https://www.ridgerun.com/gstd.")
        return False

    def test_gstd(self):
        if self.ip not in ['localhost', '127.0.0.1']:
            # bypass process check, we don't know how to start gstd remotely
{'fraction_non_alphanumeric': 0.11673984014409547, 'fraction_numerical': 0.010131712259371834, 'mean_word_length': 1.99527983816588, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2061383', 'n_tokens_mistral': 2696, 'n_tokens_neox': 2580, 'n_words': 645}