<commit_before>#!/usr/bin/python3

import sys
import os


class FileAnalyzer:
  def __init__(self):
    self._path = "";
    self._file_level = 2; # Files are 2 levels below the given path
    self._data = {}; #Data is a dictionary
    self._analyzed_path = ""
    self._dest_path = ""
    #Parameters to look for in each file.
    #We the last one (in the order that they appear in this vector)
    #is found we stop searching that file.
    self._flight_file_param = ["TIMESTAMP","EVENT","EVENTCLASS","IFPLID","REGUL"]

  def analyze_path(self,path,dest_path):
    self._data = {}; # Clear the data before doing anything
    self.analyzed_path = path;
    self._dest_path = dest_path;
    self.process_dir(path, 0);


  def complete_full_data(self,flightdata):


  def move_file(self,filepath,flightdata):
    newfilepath = os.path.join(self._dest_path, flightdata["IFPLID"])
    newfilepath = os.path.join(newfilepath, os.path.basename(filepath))
    os.renames(filepath, newfilepath);

  def process_file(self,filepath):
    flightdata = {};
    with open(filepath,'r') as f:
      for l in f:
        for param in self._flight_file_param:
          if param in l:
            flightdata[param] = l.split()[1];
            break;
        if self._flight_file_param[-1] in flightdata:
          #Last parameter is found.
          break;

    self.move_file(filepath,flightdata);
    self.complete_full_data(flightdata);


  def process_dir(self,dirpath,level):
    for entry in os.listdir(dirpath):
      fullentrypath = os.path.join(dirpath, entry);
      if level == self._file_level: # WE SHOULD BE FINDING FILES:
        if os.path.isfile(fullentrypath):
          self.process_file(fullentrypath);
        else:<commit_msg>Modified flightanalyzer<commit_after>#!/usr/bin/python3

import sys
import os


class FileAnalyzer:
  def __init__(self):
    self._path = "";
    self._file_level = 2; # Files are 2 levels below the given path
    self._data = {}; #Data is a dictionary
    self._analyzed_path = ""
    self._dest_path = ""
    #Parameters to look for in each file.
    #We the last one (in the order that they appear in this vector)
    #is found we stop searching that file.
    self._flight_file_param = ["TIMESTAMP","EVENT","EVENTCLASS","IFPLID","REGUL"]

  def analyze_path(self,path,dest_path):
    self._data = {}; # Clear the data before doing anything
    self.analyzed_path = path;
    self._dest_path = dest_path;
    self.process_dir(path, 0);


  def complete_full_data(self,flightdata):
    #Completing full data

  def move_file(self,filepath,flightdata):
    newfilepath = os.path.join(self._dest_path, flightdata["IFPLID"])
    newfilepath = os.path.join(newfilepath, os.path.basename(filepath))
    os.renames(filepath, newfilepath);

  def process_file(self,filepath):
    flightdata = {};
    with open(filepath,'r') as f:
      for l in f:
        for param in self._flight_file_param:
          if param in l:
            flightdata[param] = l.split()[1];
            break;
        if self._flight_file_param[-1] in flightdata:
          #Last parameter is found.
          break;

    self.move_file(filepath,flightdata);
    self.complete_full_data(flightdata);


  def process_dir(self,dirpath,level):
    for entry in os.listdir(dirpath):
      fullentrypath = os.path.join(dirpath, entry);
      if level == self._file_level: # WE SHOULD BE FINDING FILES:
        if os.path.isfile(fullentrypath):
          self.process_file(fullentrypath);
        else:
{'fraction_non_alphanumeric': 0.11117462132037725, 'fraction_numerical': 0.003429551300371535, 'mean_word_length': 3.137115839243499, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3198710', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1107, 'n_words': 277}