<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import socket
import ConfigParser
import time
import os
import json
from numeter.redis import RedisConnect
from numeter.storage.numeter_storage_endpoints import StorageEndpoint
from numeter.queue import server as NumeterQueueC
#import socket
import re
import logging
import sys
#import MySQLdb
import math
import hashlib
import subprocess # Clean old wsp

#Python-whisper
import whisper


import pprint # Debug (dumper)

#
# Storage
#
class Storage(object):
    def __init__(self,configFile="/etc/numeter_storage.cfg"):

        # Default configuration
        self._enable                    = False
        self._logLevel                  = "debug"
        self._log_level_stdr            = "debug"
        self._log_path                  = "/var/log/cron_numeter.log"
        self._rpc_hosts                 = ["127.0.0.1"]
        self._rpc_password              = '<PASSWORD>'
        self._host_list_file            = "/dev/shm/numeter_storage_host_list"
        self._redis_storage_port        = 6379
        self._redis_storage_timeout     = 10
        self._redis_storage_password    = None
        self._redis_storage_host        = "127.0.0.1"
        self._redis_storage_db          = 0
        self._redis_connexion           = None
        self._wsp_path                  = "/opt/numeter/wsp"
        self._wsp_path_md5_char         = 2
        self._wsp_clean_time            = 48 # 48h
        self._wsp_delete                = False
        self._host_list                 = []
        self._host_listNumber           = 0
        self._dataNumber                = 0
        self._pluginNumber              = 0
        self._sigint                    = False
        self._storage_name              = socket.gethostname()


        # Read de la conf
        self._configFile = configFile
        self.readConf()

    def startStorage(self):
        "Start storage"
        # storage enable ?
        if not self._enable:
            self._logger.warning("Numeter cron disable : "
                "configuration enable = false")
            exit(2)

        if self._redis_connexion._error:
            self._logger.critical("Redis storage connexion ERROR - "
                "Check redis access or password")
            exit(1)

        if not self._get_host_list():
            self._logger.critical("Numeter storage get host list fail")
            exit(1)

        # Time and thread param verification
        if not self.paramsVerification():
            self._logger.critical("Args verification error")
            exit(1)

        # start consumer
        self._queue_consumer = NumeterQueueC.get_rpc_server(
                                              topics=self._host_list,
                                              server=self._storage_name,
                                              endpoints=[StorageEndpoint(self)],
                                              hosts=self._rpc_hosts,
                                              password=self._rpc_password)
        try:
            self._queue_consumer.start()<commit_msg>Fix redis error. Bug added in commit <PASSWORD><commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import socket
import ConfigParser
import time
import os
import json
from numeter.redis import RedisConnect
from numeter.storage.numeter_storage_endpoints import StorageEndpoint
from numeter.queue import server as NumeterQueueC
#import socket
import re
import logging
import sys
#import MySQLdb
import math
import hashlib
import subprocess # Clean old wsp

#Python-whisper
import whisper


import pprint # Debug (dumper)

#
# Storage
#
class Storage(object):
    def __init__(self,configFile="/etc/numeter_storage.cfg"):

        # Default configuration
        self._enable                    = False
        self._logLevel                  = "debug"
        self._log_level_stdr            = "debug"
        self._log_path                  = "/var/log/cron_numeter.log"
        self._rpc_hosts                 = ["127.0.0.1"]
        self._rpc_password              = '<PASSWORD>'
        self._host_list_file            = "/dev/shm/numeter_storage_host_list"
        self._redis_storage_port        = 6379
        self._redis_storage_timeout     = 10
        self._redis_storage_password    = None
        self._redis_storage_host        = "127.0.0.1"
        self._redis_storage_db          = 0
        self._redis_connexion           = None
        self._wsp_path                  = "/opt/numeter/wsp"
        self._wsp_path_md5_char         = 2
        self._wsp_clean_time            = 48 # 48h
        self._wsp_delete                = False
        self._host_list                 = []
        self._host_listNumber           = 0
        self._dataNumber                = 0
        self._pluginNumber              = 0
        self._sigint                    = False
        self._storage_name              = socket.gethostname()


        # Read de la conf
        self._configFile = configFile
        self.readConf()

    def startStorage(self):
        "Start storage"
        # storage enable ?
        if not self._enable:
            self._logger.warning("Numeter cron disable : "
                "configuration enable = false")
            exit(2)

        self._redis_connexion = self.redisStartConnexion()

        if self._redis_connexion._error:
            self._logger.critical("Redis storage connexion ERROR - "
                "Check redis access or password")
            exit(1)

        if not self._get_host_list():
            self._logger.critical("Numeter storage get host list fail")
            exit(1)

        # Time and thread param verification
        if not self.paramsVerification():
            self._logger.critical("Args verification error")
            exit(1)

        # start consumer
        self._queue_consumer = NumeterQueueC.get_rpc_server(
                                              topics=self._host_list,
                                              server=self._storage_name,
                                              endpoints=[StorageEndpoint(self)],
                                              hosts=self._rpc_hosts,
                                              password=self._rpc_password)
        try:
            self._queue_consumer.start()
{'fraction_non_alphanumeric': 0.07311692404664355, 'fraction_numerical': 0.010400252127324299, 'mean_word_length': 1.4802657288003127, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3378015', 'n_tokens_mistral': 1689, 'n_tokens_neox': 1595, 'n_words': 358}