<commit_before>import sys
import datetime
import hashlib

from pax import __version__ as pax_version

from cax import qsub, config
from cax.task import Task

def filehash(location):
    sha = hashlib.sha512()
    with open(location, 'rb') as f:
        while True:
            block = f.read(2**10) # Magic number: one-megabyte blocks.
            if not block: break
            sha.update(block)
    return sha.hexdigest()

def verify():
    return True

def process(name, location, host):
    from pax import core
    # Grab the Run DB so we can query it
    collection = config.mongo_collection()

    # New data
    datum = {'type'       : 'processed',
             'host'       : host,
             'status'     : 'transferring',
             'location'   : location + '.root',
             'checksum'   : None,
             'pax_version': pax_version,
             'creation_time' : datetime.datetime.utcnow()}
    query = {'detector': 'tpc',
             'name'    : name}

    if collection.find_one({'detector': 'tpc',
                            'name' : name,
                            "data": { "$elemMatch": { "host": "midway-login1",
                                                      "type": "processed",
                                                      "status": "transferred"}}}) is not None:
        return

    elif collection.find_one({'detector': 'tpc',
                            'name' : name,
                            "data": { "$elemMatch": { "host": "midway-login1",
                                                      "type": "processed",
                                                      "status": "transferring"}}}) is not None:
        return

    collection.update(query,
                      {'$push': {'data': datum}})
<commit_msg>Update process.py<commit_after>import sys
import datetime
import hashlib

from pax import __version__ as pax_version

from cax import qsub, config
from cax.task import Task

def filehash(location):
    sha = hashlib.sha512()
    with open(location, 'rb') as f:
        while True:
            block = f.read(2**10) # Magic number: one-megabyte blocks.
            if not block: break
            sha.update(block)
    return sha.hexdigest()

def verify():
    return True

def process(name, location, host):
    from pax import core
    # Grab the Run DB so we can query it
    collection = config.mongo_collection()

    # New data
    datum = {'type'       : 'processed',
             'host'       : host,
             'status'     : 'transferring',
             'location'   : location + '.root',
             'checksum'   : None,
             'pax_version': pax_version,
             'creation_time' : datetime.datetime.utcnow()}
    query = {'detector': 'tpc',
             'name'    : name}

    if collection.find_one({'detector': 'tpc',
                            'name' : name,
                            "data": { "$elemMatch": { "host": "midway-login1",
                                                      "type": "processed",
                                                      "status": "transferred"}}}) is not None:
        print("Abort", name, "already processed")
        return

    elif collection.find_one({'detector': 'tpc',
                            'name' : name,
                            "data": { "$elemMatch": { "host": "midway-login1",
                                                      "type": "processed",
                                                      "status": "transferring"}}}) is not None:
        print("Abort", name, "already currently processing")
        return

    collection.update(query,
                      {'$push': {'data': datum}})

{'fraction_non_alphanumeric': 0.11798209926769732, 'fraction_numerical': 0.0043395714673176026, 'mean_word_length': 1.4073107049608355, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1736164', 'n_tokens_mistral': 973, 'n_tokens_neox': 942, 'n_words': 268}