parameters: (self)
docstring: parse log files to obtain loss and iteration data
docstring_summary: parse log files to obtain loss and iteration data
docstring_tokens: parse log files to obtain loss and iteration data
function: def _parse_log(self):
          '''parse log files to obtain loss and iteration data'''
          def obtain_log_name(log_file):
              '''get log name from log file'''
              if not os.path.isfile(log_file):
                  raise Exception('Error: log file {} does not exist.'.format(self.log_file))
              #Might need to change this to have a better representation in the future, so keep it here
              return log_file
              
          def obtain_process_id(line):
              '''get process id from line string'''
              process_id_pattern = re.compile('\[\d+\]')
              process_id_list = process_id_pattern.findall(line)
              return '0' if len(process_id_list) == 0 else process_id_list[0][1:-1]
  
          loss_pattern = re.compile('.* Iteration \d+, loss = .*')
          lr_pattern = re.compile('.* Iteration \d+, lr = .*')
          mbox_loss_pattern = re.compile('.* Train net output #\d+: mbox_loss = .*')
          detection_accuracy_pattern = re.compile('.* Test net output #\d+: detection_eval = .*')
          top1_accuracy_pattern = re.compile('.* Test net output #\d+: .*top-1 = .*')
          top5_accuracy_pattern = re.compile('.* Test net output #\d+: .*top-5 = .*')
          test_iteration_pattern = re.compile('.* Iteration \d+, Testing net \(#\d+\).*')
          for log_file in self._log_files:
              log_name = obtain_log_name(log_file)
              with open(log_file, 'r') as f:
                  for line in f.readlines():
                      if re.match(loss_pattern, line):
                          if '0' != obtain_process_id(line):
                              continue
                          self._multi_losses[log_name].append(float(line.split()[-1]))
                          self._multi_iterations[log_name].append(int(line.split()[-4][:-1]))
                      elif re.match(lr_pattern, line):
                          if '0' != obtain_process_id(line):
                              continue
                          self._multi_lrs[log_name].append(float(line.split()[-1]))
                      elif re.match(mbox_loss_pattern, line):
                          if '0' != obtain_process_id(line):
                              continue
                          self._multi_mbox_losses[log_name].append(float(line.split()[-2]))
                      elif re.match(test_iteration_pattern, line):
                          if '0' != obtain_process_id(line):
                              continue
                          self._multi_test_iterations[log_name].append(float(line.split()[-4][:-1]))
                      elif re.match(detection_accuracy_pattern, line):
                          if '0' != obtain_process_id(line):
                              continue
                          self._multi_detection_accuracies[log_name].append(float(line.split()[-1]))
                      elif re.match(top1_accuracy_pattern, line):
                          if '0' != obtain_process_id(line):
                              continue
                          self._multi_top1_accuracies[log_name].append(float(line.split()[-1]))
                      elif re.match(top5_accuracy_pattern, line):
                          if '0' != obtain_process_id(line):
                              continue
                          self._multi_top5_accuracies[log_name].append(float(line.split()[-1]))
function_tokens: def _parse_log ( self ) : def obtain_log_name ( log_file ) : '''get log name from log file''' if not os . path . isfile ( log_file ) : raise Exception ( 'Error: log file {} does not exist.' . format ( self . log_file ) ) #Might need to change this to have a better representation in the future, so keep it here return log_file def obtain_process_id ( line ) : '''get process id from line string''' process_id_pattern = re . compile ( '\[\d+\]' ) process_id_list = process_id_pattern . findall ( line ) return '0' if len ( process_id_list ) == 0 else process_id_list [ 0 ] [ 1 : - 1 ] loss_pattern = re . compile ( '.* Iteration \d+, loss = .*' ) lr_pattern = re . compile ( '.* Iteration \d+, lr = .*' ) mbox_loss_pattern = re . compile ( '.* Train net output #\d+: mbox_loss = .*' ) detection_accuracy_pattern = re . compile ( '.* Test net output #\d+: detection_eval = .*' ) top1_accuracy_pattern = re . compile ( '.* Test net output #\d+: .*top-1 = .*' ) top5_accuracy_pattern = re . compile ( '.* Test net output #\d+: .*top-5 = .*' ) test_iteration_pattern = re . compile ( '.* Iteration \d+, Testing net \(#\d+\).*' ) for log_file in self . _log_files : log_name = obtain_log_name ( log_file ) with open ( log_file , 'r' ) as f : for line in f . readlines ( ) : if re . match ( loss_pattern , line ) : if '0' != obtain_process_id ( line ) : continue self . _multi_losses [ log_name ] . append ( float ( line . split ( ) [ - 1 ] ) ) self . _multi_iterations [ log_name ] . append ( int ( line . split ( ) [ - 4 ] [ : - 1 ] ) ) elif re . match ( lr_pattern , line ) : if '0' != obtain_process_id ( line ) : continue self . _multi_lrs [ log_name ] . append ( float ( line . split ( ) [ - 1 ] ) ) elif re . match ( mbox_loss_pattern , line ) : if '0' != obtain_process_id ( line ) : continue self . _multi_mbox_losses [ log_name ] . append ( float ( line . split ( ) [ - 2 ] ) ) elif re . match ( test_iteration_pattern , line ) : if '0' != obtain_process_id ( line ) : continue self . _multi_test_iterations [ log_name ] . append ( float ( line . split ( ) [ - 4 ] [ : - 1 ] ) ) elif re . match ( detection_accuracy_pattern , line ) : if '0' != obtain_process_id ( line ) : continue self . _multi_detection_accuracies [ log_name ] . append ( float ( line . split ( ) [ - 1 ] ) ) elif re . match ( top1_accuracy_pattern , line ) : if '0' != obtain_process_id ( line ) : continue self . _multi_top1_accuracies [ log_name ] . append ( float ( line . split ( ) [ - 1 ] ) ) elif re . match ( top5_accuracy_pattern , line ) : if '0' != obtain_process_id ( line ) : continue self . _multi_top5_accuracies [ log_name ] . append ( float ( line . split ( ) [ - 1 ] ) )