<commit_before>"""
daemon_process.py

This module contains functions that help implement the datawarehouse 
daemon mode.

"""
import shutil
from multiprocessing import Process, Queue
from Queue import Empty
from datetime import datetime, timedelta
import os
import re
import subprocess
import getpass
from glob import glob

from time import sleep
from astrodata import Lookups
from astrodata.adutils import ksutil as ks
from astrodata.adutils.dwutil.dwsettings import ingest_sources
from astrodata.adutils.dwutil.dwsettings import warehouse_packages
from astrodata.adutils.dwutil.dwsettings import package_dict

POLL_RESOLUTION = .5 # in seconds
def clear_directory(path):
    print "Removing contents of %s" % path
    for fil in os.listdir(path):
        if os.path.islink(fil):
            shutil.remove(fil)
        elif os.path.isdir(fil):
            shutil.rmtree(fil)
        elif os.path.isfile(fil):
            os.remove(fil)
            
class ShelfWatch(object):
    source = None
    frequency = 10
    ingest_package = None
    publish_package = None
    prefix = None
    _last_check = None
    _current_manifests = None
    
    def __init__(self, source = None):
        self._current_manifests = []
        if source:
            self.source = source
            if "elements" in self.source:
                elements = self.source["elements"]
            else:
                elements = {}
            if "shelf_name" not in elements:
                elements["shelf_name"] = source["ingest_shelf"]
            if "frequency" in source:
                self.frequency = source["frequency"]
            if "user" not in source:
                source["user"] = getpass.getuser()
            if "home" not in source:<commit_msg>small fix<commit_after>"""
daemon_process.py

This module contains functions that help implement the datawarehouse 
daemon mode.

"""
import shutil
from multiprocessing import Process, Queue
from Queue import Empty
from datetime import datetime, timedelta
import os
import re
import subprocess
import getpass
from glob import glob

from time import sleep
from astrodata import Lookups
from astrodata.adutils import ksutil as ks
from astrodata.adutils.dwutil.dwsettings import ingest_sources
from astrodata.adutils.dwutil.dwsettings import warehouse_packages
from astrodata.adutils.dwutil.dwsettings import package_dict

POLL_RESOLUTION = .5 # in seconds
def clear_directory(path):
    print "Removing contents of %s" % path
    for fil in os.listdir(path):
        if os.path.islink(fil):
            os.remove(fil)
        elif os.path.isdir(fil):
            shutil.rmtree(fil)
        elif os.path.isfile(fil):
            os.remove(fil)
            
class ShelfWatch(object):
    source = None
    frequency = 10
    ingest_package = None
    publish_package = None
    prefix = None
    _last_check = None
    _current_manifests = None
    
    def __init__(self, source = None):
        self._current_manifests = []
        if source:
            self.source = source
            if "elements" in self.source:
                elements = self.source["elements"]
            else:
                elements = {}
            if "shelf_name" not in elements:
                elements["shelf_name"] = source["ingest_shelf"]
            if "frequency" in source:
                self.frequency = source["frequency"]
            if "user" not in source:
                source["user"] = getpass.getuser()
            if "home" not in source:
{'fraction_non_alphanumeric': 0.0757314974182444, 'fraction_numerical': 0.0017211703958691911, 'mean_word_length': 2.466202783300199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '522609', 'n_tokens_mistral': 1003, 'n_tokens_neox': 972, 'n_words': 300}