<commit_before>import os
import time
import argparse
import shutil
import subprocess
import warnings
import sys

import numpy as np
import pandas as pd

from src import SPclean
import src.Paths as PATH


def parser():
  '''
  Command-line options
  '''

  parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,description="The program creates a DataBase of the events in the observation.")
  parser.add_argument('id_obs', help='Observation ID')
  parser.add_argument('-conf', help="Run over confirmation observations.",action='store_true')
  parser.add_argument('-load_DB', help="Load events from a DataBase instead of text files.",action='store_true')
  parser.add_argument('--debug', help="Debug mode: shows all the warnings.",action='store_true')
  args = parser.parse_args()
  
  return args


def main(PATH):
  #warnings.filterwarnings('error', category=FutureWarning)

  args = parser()
  if args.conf:
    PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, 'confirmations')
    PATH.RAW_FOLDER = os.path.join(PATH.RAW_FOLDER, 'confirmations')
  
  PATH.WRK_FOLDER = os.path.join(PATH.WRK_FOLDER, args.id_obs)
  PATH.TMP_FOLDER = os.path.join(PATH.TMP_FOLDER, args.id_obs)
  PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, args.id_obs)

  if os.path.isdir(PATH.WRK_FOLDER): shutil.rmtree(PATH.WRK_FOLDER)
  os.makedirs(PATH.WRK_FOLDER)
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp'))
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp/candidates'))
  if os.path.isdir(PATH.TMP_FOLDER): shutil.rmtree(PATH.TMP_FOLDER)
  os.makedirs(PATH.TMP_FOLDER)
  if os.path.isdir(os.path.join(PATH.OBS_FOLDER, 'sp')): shutil.rmtree(os.path.join(PATH.OBS_FOLDER, 'sp'))
  
  with open(os.path.join(PATH.WRK_FOLDER, 'sp/log.txt'), 'w') as f:
    sys.stdout = f
  with open(os.path.join(PATH.WRK_FOLDER, 'sp/ERROR_log.txt'), 'w') as f:
    sys.stderr = f

  scriptFolder = os.path.dirname(os.path.realpath(__file__))
  git_folder = os.path.join(scriptFolder, '.git')
  vers = subprocess.check_output(['git','--git-dir',git_folder,'describe','--tags','--abbrev=0','--always']).strip()
  print "L-SpS version used: {}".format(vers)
  args.vers = vers

  time0 = time.time()
  print "The DataBase is being created."

  try:
    SPclean.main(args, PATH)
    print "The DataBase has been created."
    print "Time spent: {:.2f} s.".format(time.time() - time0)

  finally:
    shutil.copytree(os.path.join(PATH.WRK_FOLDER, 'sp'), os.path.join(PATH.OBS_FOLDER, 'sp'))
    shutil.rmtree(PATH.WRK_FOLDER)
    shutil.rmtree(PATH.TMP_FOLDER)

  return


if __name__ == '__main__':
  main(PATH)<commit_msg>Changes for Pipeline v2<commit_after>import os
import time
import argparse
import shutil
import subprocess
import warnings
import sys

import numpy as np
import pandas as pd

from src import SPclean
import src.Paths as PATH


def parser():
  '''
  Command-line options
  '''

  parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,description="The program creates a DataBase of the events in the observation.")
  parser.add_argument('id_obs', help='Observation ID')
  parser.add_argument('-conf', help="Run over confirmation observations.",action='store_true')
  parser.add_argument('-load_DB', help="Load events from a DataBase instead of text files.",action='store_true')
  parser.add_argument('--debug', help="Debug mode: shows all the warnings.",action='store_true')
  args = parser.parse_args()
  
  return args


def main(PATH):
  #warnings.filterwarnings('error', category=FutureWarning)

  args = parser()
  if args.conf:
    PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, 'confirmations')
    PATH.RAW_FOLDER = os.path.join(PATH.RAW_FOLDER, 'confirmations')
  
  PATH.WRK_FOLDER = os.path.join(PATH.WRK_FOLDER, args.id_obs)
  PATH.TMP_FOLDER = os.path.join(PATH.TMP_FOLDER, args.id_obs)
  PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, args.id_obs)

  if os.path.isdir(PATH.WRK_FOLDER): shutil.rmtree(PATH.WRK_FOLDER)
  os.makedirs(PATH.WRK_FOLDER)
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp'))
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp/candidates'))
  if os.path.isdir(PATH.TMP_FOLDER): shutil.rmtree(PATH.TMP_FOLDER)
  os.makedirs(PATH.TMP_FOLDER)
  if os.path.isdir(os.path.join(PATH.OBS_FOLDER, 'sp')): shutil.rmtree(os.path.join(PATH.OBS_FOLDER, 'sp'))
  
  with open(os.path.join(PATH.WRK_FOLDER, 'sp/log.txt'), 'w') as f:
    sys.stdout = f
  with open(os.path.join(PATH.WRK_FOLDER, 'sp/ERROR_log.txt'), 'w') as f:
    sys.stderr = f

  scriptFolder = os.path.dirname(os.path.realpath(__file__))
  git_folder = os.path.join(scriptFolder, '.git')
  vers = subprocess.check_output(['git','--git-dir',git_folder,'describe','--tags','--abbrev=0','--always']).strip()
  print "L-SpS version used: {}".format(vers)
  args.vers = vers

  time0 = time.time()
  print "The DataBase is being created."

  try:
    SPclean.main(args)
    print "The DataBase has been created."
    print "Time spent: {:.2f} s.".format(time.time() - time0)

  finally:
    shutil.copytree(os.path.join(PATH.WRK_FOLDER, 'sp'), os.path.join(PATH.OBS_FOLDER, 'sp'))
    shutil.rmtree(PATH.WRK_FOLDER)
    shutil.rmtree(PATH.TMP_FOLDER)

  return


if __name__ == '__main__':
  main(PATH)
{'fraction_non_alphanumeric': 0.1492166602980512, 'fraction_numerical': 0.001719526175009553, 'mean_word_length': 4.647249190938512, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3640703', 'n_tokens_mistral': 2025, 'n_tokens_neox': 1986, 'n_words': 361}