<commit_before>import daemon
from lockfile.pidlockfile import PIDLockFile
from lockfile import AlreadyLocked
from collections import defaultdict
import logging
import os
import datetime
import sys
import time
import traceback
import yaml

from lobster import job, util, cmssw

import work_queue as wq

def get_lock(workdir):
    pidfile = PIDLockFile(os.path.join(workdir, 'lobster.pid'), timeout=-1)
    try:
        pidfile.acquire()
    except AlreadyLocked:
            print "Another instance of lobster is accessing {0}".format(workdir)
            raise
    pidfile.break_lock()
    return pidfile

def kill(args):
    with open(args.configfile) as configfile:
        config = yaml.load(configfile)

    workdir = config['workdir']
    util.register_checkpoint(workdir, 'KILLED', 'PENDING')

def cleanup(args):
    with open(args.configfile) as configfile:
        config = yaml.load(configfile)

    store = cmssw.jobit.JobitStore(config)
    config = job.apply_matching(config)
    outputformats = defaultdict(list)
    deleted_files = 0
    for cfg in config['tasks']:
        good_files = set()
        label = cfg['label']
        for id in store.finished_jobs(label):
            for base, ext in [os.path.splitext(o) for o in cfg['outputs']]:
                output_format = cfg.get("output format", "{base}_{id}.{ext}")
                good_files.add(output_format.format(base=base, id=id[0], ext=ext[1:]))

        for dirpath, dirnames, filenames in os.walk(os.path.join(config['stageout location'], label)):
            print 'Looking for output files to cleanup in {0}...'.format(label)

            files = set(filenames)
            missing = good_files - files
            extra = files - good_files

            for file in missing:
                print 'Warning!  Expected to find {0}, but it is missing!'.format(file)
            for file in extra:
                print 'Found output from failed job: deleting {0}...'.format(file)
                deleted_files += 1
                os.remove(os.path.join(dirpath, file))

    conjugation = 's' if deleted_files != 1 else ''
    print 'Finished cleaning; found {0} file{1} to clean up.'.format(deleted_files, conjugation)
<commit_msg>Removing unnecessary dict.<commit_after>import daemon
from lockfile.pidlockfile import PIDLockFile
from lockfile import AlreadyLocked
import logging
import os
import datetime
import sys
import time
import traceback
import yaml

from lobster import job, util, cmssw

import work_queue as wq

def get_lock(workdir):
    pidfile = PIDLockFile(os.path.join(workdir, 'lobster.pid'), timeout=-1)
    try:
        pidfile.acquire()
    except AlreadyLocked:
            print "Another instance of lobster is accessing {0}".format(workdir)
            raise
    pidfile.break_lock()
    return pidfile

def kill(args):
    with open(args.configfile) as configfile:
        config = yaml.load(configfile)

    workdir = config['workdir']
    util.register_checkpoint(workdir, 'KILLED', 'PENDING')

def cleanup(args):
    with open(args.configfile) as configfile:
        config = yaml.load(configfile)

    store = cmssw.jobit.JobitStore(config)
    config = job.apply_matching(config)
    deleted_files = 0
    for cfg in config['tasks']:
        good_files = set()
        label = cfg['label']
        for id in store.finished_jobs(label):
            for base, ext in [os.path.splitext(o) for o in cfg['outputs']]:
                output_format = cfg.get("output format", "{base}_{id}.{ext}")
                good_files.add(output_format.format(base=base, id=id[0], ext=ext[1:]))

        for dirpath, dirnames, filenames in os.walk(os.path.join(config['stageout location'], label)):
            print 'Looking for output files to cleanup in {0}...'.format(label)

            files = set(filenames)
            missing = good_files - files
            extra = files - good_files

            for file in missing:
                print 'Warning!  Expected to find {0}, but it is missing!'.format(file)
            for file in extra:
                print 'Found output from failed job: deleting {0}...'.format(file)
                deleted_files += 1
                os.remove(os.path.join(dirpath, file))

    conjugation = 's' if deleted_files != 1 else ''
    print 'Finished cleaning; found {0} file{1} to clean up.'.format(deleted_files, conjugation)

{'fraction_non_alphanumeric': 0.10593900481540931, 'fraction_numerical': 0.005503324925475808, 'mean_word_length': 2.744206008583691, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '58191', 'n_tokens_mistral': 1329, 'n_tokens_neox': 1282, 'n_words': 372}