<commit_before>#! /usr/bin/env python

import psycopg2

import time
import datetime
import os
import os.path
import shutil

import logging

logger = logging.getLogger("monitor")

import socorro.lib.util

def directoryJudgedDeletable (config, pathname, subDirectoryList, fileList):
  if not (subDirectoryList or fileList) and pathname != config.storageRoot: #if both directoryList and fileList are empty
    #select an ageLimit from two options based on the if target directory name has a prefix of "dumpDirPrefix"
    ageLimit = (config.dateDirDelta, config.dumpDirDelta)[os.path.basename(pathname).startswith(config.dumpDirPrefix)]
    return (datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(os.path.getmtime(pathname))) > ageLimit
  return False

def ignoreDuplicateDatabaseInsert (exceptionType, exception, tracebackInfo):
  return exceptionType is psycopg2.IntegrityError

def archiveCompletedJobFiles (config, jsonPathname, uuid, newFileExtension):
  logger.debug("archiving %s", jsonPathname)
  newJsonPathname = ("%s/%s%s.%s" % (config.saveMinidumpsTo, uuid, config.jsonFileSuffix, newFileExtension)).replace('//','/')
  try:
    shutil.move(jsonPathname, newJsonPathname)
  except:
    socorro.lib.util.reportExceptionAndContinue(logger)
  if config.debug:
    try:
      f = open(jsonPathname)
      f.close()
      logger.error("removed failed - %s", jsonPathname)
    except:
      logger.debug("remove succeeded - %s", jsonPathname)
    try:
      f = open(newJsonPathname)
      f.close()
      logger.debug("arrival succeeded - %s", newJsonPathname)
    except:
      logger.error("arrival failed - %s", newJsonPathname)      
      
  try:
    dumpPathname = "%s%s" % (jsonPathname[:-len(config.jsonFileSuffix)], config.dumpFileSuffix)
    newDumpPathname = ("%s/%s%s.%s" % (config.saveMinidumpsTo, uuid, config.dumpFileSuffix, newFileExtension)).replace('//','/')
    logger.debug("archiving %s", dumpPathname)
    os.rename(dumpPathname, newDumpPathname)
  except:
    socorro.lib.util.reportExceptionAndContinue(logger)
  if config.debug:
    try:
      f = open(dumpPathname)
      f.close()
      logger.error("removed failed - %s", dumpPathname)
    except:
      logger.debug("remove succeeded - %s", dumpPathname)
    try:
      f = open(newDumpPathname)
      f.close()
      logger.debug("arrival succeeded - %s", newDumpPathname)
    except:
      logger.error("arrival failed - %s", newDumpPathname)    
      
def deleteCompletedJobFiles (config, jsonPathname, unused1, unused2):
  logger.debug("deleting %s", jsonPathname)
  try:
    os.remove(jsonPathname)
  except:
    socorro.lib.util.reportExceptionAndContinue(logger) <commit_msg>changed os.rename to shutil.move in both places it was used rather than just once...<commit_after>#! /usr/bin/env python

import psycopg2

import time
import datetime
import os
import os.path
import shutil

import logging

logger = logging.getLogger("monitor")

import socorro.lib.util

def directoryJudgedDeletable (config, pathname, subDirectoryList, fileList):
  if not (subDirectoryList or fileList) and pathname != config.storageRoot: #if both directoryList and fileList are empty
    #select an ageLimit from two options based on the if target directory name has a prefix of "dumpDirPrefix"
    ageLimit = (config.dateDirDelta, config.dumpDirDelta)[os.path.basename(pathname).startswith(config.dumpDirPrefix)]
    return (datetime.datetime.utcnow() - datetime.datetime.utcfromtimestamp(os.path.getmtime(pathname))) > ageLimit
  return False

def ignoreDuplicateDatabaseInsert (exceptionType, exception, tracebackInfo):
  return exceptionType is psycopg2.IntegrityError

def archiveCompletedJobFiles (config, jsonPathname, uuid, newFileExtension):
  logger.debug("archiving %s", jsonPathname)
  newJsonPathname = ("%s/%s%s.%s" % (config.saveMinidumpsTo, uuid, config.jsonFileSuffix, newFileExtension)).replace('//','/')
  try:
    shutil.move(jsonPathname, newJsonPathname)
  except:
    socorro.lib.util.reportExceptionAndContinue(logger)
  if config.debug:
    try:
      f = open(jsonPathname)
      f.close()
      logger.error("removed failed - %s", jsonPathname)
    except:
      logger.debug("remove succeeded - %s", jsonPathname)
    try:
      f = open(newJsonPathname)
      f.close()
      logger.debug("arrival succeeded - %s", newJsonPathname)
    except:
      logger.error("arrival failed - %s", newJsonPathname)      
      
  try:
    dumpPathname = "%s%s" % (jsonPathname[:-len(config.jsonFileSuffix)], config.dumpFileSuffix)
    newDumpPathname = ("%s/%s%s.%s" % (config.saveMinidumpsTo, uuid, config.dumpFileSuffix, newFileExtension)).replace('//','/')
    logger.debug("archiving %s", dumpPathname)
    shutil.move(dumpPathname, newDumpPathname)
  except:
    socorro.lib.util.reportExceptionAndContinue(logger)
  if config.debug:
    try:
      f = open(dumpPathname)
      f.close()
      logger.error("removed failed - %s", dumpPathname)
    except:
      logger.debug("remove succeeded - %s", dumpPathname)
    try:
      f = open(newDumpPathname)
      f.close()
      logger.debug("arrival succeeded - %s", newDumpPathname)
    except:
      logger.error("arrival failed - %s", newDumpPathname)    
      
def deleteCompletedJobFiles (config, jsonPathname, unused1, unused2):
  logger.debug("deleting %s", jsonPathname)
  try:
    os.remove(jsonPathname)
  except:
    socorro.lib.util.reportExceptionAndContinue(logger) 
{'fraction_non_alphanumeric': 0.11609935602575897, 'fraction_numerical': 0.0014719411223551057, 'mean_word_length': 4.113828786453434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5494665', 'n_tokens_mistral': 1730, 'n_tokens_neox': 1711, 'n_words': 383}