<commit_before>#!/usr/bin/python

import sys
from merge import loadQueueFile
import logging
import subprocess
from subprocess import PIPE, STDOUT
import os
import os.path

logging.basicConfig(filename='/var/log/cloud.log',level=logging.DEBUG, format='%(asctime)s %(message)s')

# first commandline argument should be the file to process
if ( len(sys.argv) != 2 ):
    print "Invalid usage"
    sys.exit(1)

# FIXME we should get this location from a configuration class
filePath = "/var/cache/cloud/%s" % sys.argv[1]
if not (os.path.isfile(filePath) and os.access(filePath, os.R_OK)):
    print "You are telling me to process %s, but i can't access it" % filePath
    sys.exit(1)

# If the command line json file is unprocessed process it
# This is important or, the control interfaces will get deleted!
if os.path.isfile("filePath/%s" % "cmd_line.json"):
    qf = loadQueueFile()
    qf.setFile("cmd_line.json")
    qf.load(None)

qf = loadQueueFile()
qf.setFile(sys.argv[1])
qf.load(None)

# Converge
run = subprocess.Popen(["/opt/cloud/bin/configure.py"],
                       stdin=PIPE, stdout=PIPE, stderr=PIPE)
run.wait()

if run.returncode:
    print "stdout >>"
    print run.stdout.read()
    print "stderr >>"
    print run.stderr.read()
else:
    print "Convergence is achieved - you have been assimilated!"

sys.exit(run.returncode)<commit_msg>We actually don't need all the process stuff<commit_after>#!/usr/bin/python

import sys
from merge import loadQueueFile
import logging
import subprocess
from subprocess import PIPE, STDOUT
import os
import os.path
import configure

logging.basicConfig(filename='/var/log/cloud.log',level=logging.DEBUG, format='%(asctime)s %(message)s')

# first commandline argument should be the file to process
if ( len(sys.argv) != 2 ):
    print "Invalid usage"
    sys.exit(1)

# FIXME we should get this location from a configuration class
filePath = "/var/cache/cloud/%s" % sys.argv[1]
if not (os.path.isfile(filePath) and os.access(filePath, os.R_OK)):
    print "You are telling me to process %s, but i can't access it" % filePath
    sys.exit(1)

# If the command line json file is unprocessed process it
# This is important or, the control interfaces will get deleted!
if os.path.isfile("filePath/%s" % "cmd_line.json"):
    qf = loadQueueFile()
    qf.setFile("cmd_line.json")
    qf.load(None)

qf = loadQueueFile()
qf.setFile(sys.argv[1])
qf.load(None)

# Converge
returncode = configure.main([])

sys.exit(returncode)
{'fraction_non_alphanumeric': 0.12323232323232323, 'fraction_numerical': 0.00404040404040404, 'mean_word_length': 4.190775681341719, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5450278', 'n_tokens_mistral': 841, 'n_tokens_neox': 826, 'n_words': 247}