<commit_before>import threading
import logging
import os

from biomaj.process.process import Process, DrmaaProcess, DockerProcess
from biomaj.mongo_connector import MongoConnector

class MetaProcess(threading.Thread):
    '''
    Meta process in biomaj process workflow. Meta processes are executed in parallel.

    Each meta process defined a list of Process to execute sequentially
    '''

    def __init__(self, bank, metas, meta_status=None, meta_data={}, simulate=False):
      '''
      Creates a meta process thread

      :param bank: Bank
      :type bank: :class:`biomak.bank`
      :param meta: list of meta processes to execute in thread
      :type meta: list of str
      :param meta_status: initial status of the meta processes
      :type meta_status: bool
      :param simulate: does not execute process
      :type simulate: bool
      '''
      threading.Thread.__init__(self)
      self._lock = None
      self.kill_received = False
      self.workflow = None
      self.simulate = simulate
      self.bank = bank
      self.metas = metas
      self.meta_data = meta_data
      self.meta_status = {}
      for meta in self.metas:
        self.meta_status[meta] = {}

      if meta_status is not None:
        self.meta_status = meta_status

      self._stopevent = threading.Event( )

      self.bmaj_env = os.environ.copy()
      #self.bmaj_env = {}
      # Copy all config from bank


      self.bmaj_only_env = {}
      #The root directory where all databases are stored.
      #If your data is not stored under one directory hirearchy
      #you can override this value in the database properties file.
      for conf in dict(self.bank.config.config_bank.items('GENERAL')):
        self.bmaj_env[conf] = self.bank.config.config_bank.get('GENERAL', conf)
        if self.bmaj_env[conf] is None:
          self.bmaj_env[conf] = ''
          self.bmaj_only_env[conf] = self.bmaj_env[conf]

      self.bmaj_env['dbname'] = self.bank.name
      self.bmaj_only_env['dbname'] = self.bmaj_env['dbname']

      self.bmaj_env['datadir'] = self.bank.config.get('data.dir')
      self.bmaj_only_env['datadir'] = self.bmaj_env['datadir']

      self.bmaj_env['data.dir'] = self.bmaj_env['datadir']
      self.bmaj_only_env['data.dir'] = self.bmaj_env['data.dir']

      self.bmaj_env['mailadmin'] = self.bank.config.get('mail.admin')
      self.bmaj_only_env['mailadmin'] = self.bmaj_env['mailadmin']

      if self.bank.config.get('mail.smtp.host'):
        self.bmaj_env['mailsmtp'] = self.bank.config.get('mail.smtp.host')
        self.bmaj_only_env['mailsmtp'] = self.bmaj_env['mailsmtp']

      self.bmaj_env['processdir'] = self.bank.config.get('process.dir',default='')
      self.bmaj_only_env['processdir'] = self.bmaj_env['processdir']

      if 'PATH' in self.bmaj_env:
        self.bmaj_env['PATH'] += ':' + self.bmaj_env['processdir']
        self.bmaj_only_env['PATH'] = self.bmaj_env['PATH']
      else:
        self.bmaj_env['PATH'] = self.bmaj_env['processdir']+':/usr/local/bin:/usr/sbin:/usr/bin'
        self.bmaj_only_env['PATH'] = self.bmaj_env['PATH']

      self.bmaj_env['PATH_PROCESS_BIOMAJ'] = self.bank.config.get('process.dir')

      # Set some session specific env
      if self.bank.session is not None:
        self.bmaj_env['offlinedir'] = self.bank.session.get_offline_directory()
        self.bmaj_only_env['offlinedir'] = self.bmaj_env['offlinedir']

        self.bmaj_env['dirversion'] = self.bank.config.get('dir.version')
        self.bmaj_only_env['dirversion'] = self.bmaj_env['dirversion']

        self.bmaj_env['noextract'] = self.bank.config.get('no.extract')
        if self.bmaj_env['noextract'] is None:
          self.bmaj_env['noextract'] = ''
        self.bmaj_only_env['noextract'] = self.bmaj_env['noextract']
<commit_msg>test if mail is set before putting it in env<commit_after>import threading
import logging
import os

from biomaj.process.process import Process, DrmaaProcess, DockerProcess
from biomaj.mongo_connector import MongoConnector

class MetaProcess(threading.Thread):
    '''
    Meta process in biomaj process workflow. Meta processes are executed in parallel.

    Each meta process defined a list of Process to execute sequentially
    '''

    def __init__(self, bank, metas, meta_status=None, meta_data={}, simulate=False):
      '''
      Creates a meta process thread

      :param bank: Bank
      :type bank: :class:`biomak.bank`
      :param meta: list of meta processes to execute in thread
      :type meta: list of str
      :param meta_status: initial status of the meta processes
      :type meta_status: bool
      :param simulate: does not execute process
      :type simulate: bool
      '''
      threading.Thread.__init__(self)
      self._lock = None
      self.kill_received = False
      self.workflow = None
      self.simulate = simulate
      self.bank = bank
      self.metas = metas
      self.meta_data = meta_data
      self.meta_status = {}
      for meta in self.metas:
        self.meta_status[meta] = {}

      if meta_status is not None:
        self.meta_status = meta_status

      self._stopevent = threading.Event( )

      self.bmaj_env = os.environ.copy()
      #self.bmaj_env = {}
      # Copy all config from bank


      self.bmaj_only_env = {}
      #The root directory where all databases are stored.
      #If your data is not stored under one directory hirearchy
      #you can override this value in the database properties file.
      for conf in dict(self.bank.config.config_bank.items('GENERAL')):
        self.bmaj_env[conf] = self.bank.config.config_bank.get('GENERAL', conf)
        if self.bmaj_env[conf] is None:
          self.bmaj_env[conf] = ''
          self.bmaj_only_env[conf] = self.bmaj_env[conf]

      self.bmaj_env['dbname'] = self.bank.name
      self.bmaj_only_env['dbname'] = self.bmaj_env['dbname']

      self.bmaj_env['datadir'] = self.bank.config.get('data.dir')
      self.bmaj_only_env['datadir'] = self.bmaj_env['datadir']

      self.bmaj_env['data.dir'] = self.bmaj_env['datadir']
      self.bmaj_only_env['data.dir'] = self.bmaj_env['data.dir']

      if self.bank.config.get('mail.admin'):
          self.bmaj_env['mailadmin'] = self.bank.config.get('mail.admin')
          self.bmaj_only_env['mailadmin'] = self.bmaj_env['mailadmin']

      if self.bank.config.get('mail.smtp.host'):
        self.bmaj_env['mailsmtp'] = self.bank.config.get('mail.smtp.host')
        self.bmaj_only_env['mailsmtp'] = self.bmaj_env['mailsmtp']

      self.bmaj_env['processdir'] = self.bank.config.get('process.dir',default='')
      self.bmaj_only_env['processdir'] = self.bmaj_env['processdir']

      if 'PATH' in self.bmaj_env:
        self.bmaj_env['PATH'] += ':' + self.bmaj_env['processdir']
        self.bmaj_only_env['PATH'] = self.bmaj_env['PATH']
      else:
        self.bmaj_env['PATH'] = self.bmaj_env['processdir']+':/usr/local/bin:/usr/sbin:/usr/bin'
        self.bmaj_only_env['PATH'] = self.bmaj_env['PATH']

      self.bmaj_env['PATH_PROCESS_BIOMAJ'] = self.bank.config.get('process.dir')

      # Set some session specific env
      if self.bank.session is not None:
        self.bmaj_env['offlinedir'] = self.bank.session.get_offline_directory()
        self.bmaj_only_env['offlinedir'] = self.bmaj_env['offlinedir']

        self.bmaj_env['dirversion'] = self.bank.config.get('dir.version')
        self.bmaj_only_env['dirversion'] = self.bmaj_env['dirversion']

        self.bmaj_env['noextract'] = self.bank.config.get('no.extract')
        if self.bmaj_env['noextract'] is None:
          self.bmaj_env['noextract'] = ''
        self.bmaj_only_env['noextract'] = self.bmaj_env['noextract']

{'fraction_non_alphanumeric': 0.12907856956408248, 'fraction_numerical': 0.0, 'mean_word_length': 3.093482905982906, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4599435', 'n_tokens_mistral': 2673, 'n_tokens_neox': 2596, 'n_words': 497}