<commit_before>#!/usr/bin/python
# --------------------------------------------------------------------------------------------------
# This is the core of managing download requests. It proceeds in five steps. The order is important
# to make sure not to loose any requests.
#
# status of downloading request: the general idea is to move each download request through 3
#                                distinct states 0, 1, 2
#
#    0 - request is entered initially with status 0, meaning it is new and has not been acted upon
#    1 - once the request has been put in the queue for downloading (job submitted)
#    2 - the request has been acted upon and the download request is not anymore in the queue
#        there is no test of success here, the system assumes that the request was successful
#
#
# 1. PREP    - preparations to find all ongoing download requests in the queue (condor)
#
# 2. PART 0  - query the database to find download request in status 2 (see explanation below)
#              requests found in status 2 will be deleted from the database
#
# 3. PART 1  - query the database to find download request in status 1 (see explanation below)
#              requests found in status 1 and not anymore in our queue will be moved to status 2
#
# 4. PART 2  - query the database to find download request in status 0 (see explanation below)
#              requests found in status 0 will be processed and submitted to the queue
#
# 5. MONITOR - provide basic monitoring status
#
# TODO -- [ we need a step 6 in which old logfiles will be cleared ]
#
# --------------------------------------------------------------------------------------------------
import os, MySQLdb, time

# define all relevant variable in python
SMARTCACHE_DATA="/mnt/hadoop/cms/store/user/paus"
SMARTCACHE_DIR="/usr/local/DynamicData/SmartCache"
SMARTCACHE_LOGDIR="/var/log/DynamicData/SmartCache"<commit_msg>Fix bug in reviewRequests.py.<commit_after>#!/usr/bin/python
# --------------------------------------------------------------------------------------------------
# This is the core of managing download requests. It proceeds in five steps. The order is important
# to make sure not to loose any requests.
#
# status of downloading request: the general idea is to move each download request through 3
#                                distinct states 0, 1, 2
#
#    0 - request is entered initially with status 0, meaning it is new and has not been acted upon
#    1 - once the request has been put in the queue for downloading (job submitted)
#    2 - the request has been acted upon and the download request is not anymore in the queue
#        there is no test of success here, the system assumes that the request was successful
#
#
# 1. PREP    - preparations to find all ongoing download requests in the queue (condor)
#
# 2. PART 0  - query the database to find download request in status 2 (see explanation below)
#              requests found in status 2 will be deleted from the database
#
# 3. PART 1  - query the database to find download request in status 1 (see explanation below)
#              requests found in status 1 and not anymore in our queue will be moved to status 2
#
# 4. PART 2  - query the database to find download request in status 0 (see explanation below)
#              requests found in status 0 will be processed and submitted to the queue
#
# 5. MONITOR - provide basic monitoring status
#
# TODO -- [ we need a step 6 in which old logfiles will be cleared ]
#
# --------------------------------------------------------------------------------------------------
import os, MySQLdb, time
import datetime
from   datetime import date, timedelta

# define all relevant variable in python
SMARTCACHE_DATA="/mnt/hadoop/cms/store/user/paus"
SMARTCACHE_DIR="/usr/local/DynamicData/SmartCache"
SMARTCACHE_LOGDIR="/var/log/DynamicData/SmartCache"
{'fraction_non_alphanumeric': 0.15283165244375485, 'fraction_numerical': 0.012412723041117145, 'mean_word_length': 2.1319838056680163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3292505', 'n_tokens_mistral': 944, 'n_tokens_neox': 881, 'n_words': 497}