<commit_before>#!/usr/bin/env python2
"""
WARNING!  This file is auto updated from the node manager.  Any changes will
be lost when the client disconnects and reconnects.

This file is compatible with python2 and python3.

Theory of operation
1. Read the config file getting
   - Server and port to connect too
   - What arrays are available to use
2. Connect to the server
3. Wait for a command request
4. If server connection goes away or haven't received ping, close connection
   and try to periodically re-establish communication
Command requests include:
ping - check to see if the connection is working
arrays - Return array information on what is available to test
running - Return which arrays are currently running tests
job_create - Submit a new test to run (Creates a new process)
jobs - Return job state information on all submitted jobs
job - Return job state information about a specific job
job_delete - Delete the specific job request freeing resources
job_completion - Retrieve the exit code and log for specified job
Service which runs a test(s) on the same box as service
 See: https://github.com/tasleson/lsm-ci/blob/master/LICENSE
"""

import string
import random
import yaml
import os
import json
import sys
from subprocess import call
from multiprocessing import Process
import testlib
import time
import traceback
import tempfile
import shutil

jobs = {}
config = {}

STARTUP_CWD = ""

NODE = None


def _lcall(command, job_id):
    """
    Call an executable and return a tuple of exitcode, stdout&stderr
    """

    # Write output to a file so we can see what's going on while it's running
    f = '/tmp/%s.out' % job_id

    with open(f, 'w', buffering=1) as log:  # Max buffer 1 line (text mode)
        exit_value = call(command, stdout=log, stderr=log)
    return exit_value, f


def _file_name(job_id):
    # If this log directory is located in /tmp, the system may remove the
    # directory after a while, making us fail to log when needed.
    log_dir = config["LOGDIR"]
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    base = '%s/%s' % (log_dir, job_id)
    return base + ".out"


def _run_command(job_id, args):
    cmd = [config["PROGRAM"]]
    log_dir = config["LOGDIR"]

    cmd.extend(args)

    (ec, output_file) = _lcall(cmd, job_id)
    log = _file_name(job_id)

    # Read in output file in it's entirety
    with open(output_file, 'r') as o:
        out = o.read()

    # Delete file to prevent /tmp from filling up
    os.remove(output_file)

    with open(log, 'w') as error_file:
        error_file.write(yaml.dump(dict(EC=str(ec), OUTPUT=out)))
        error_file.flush()

    # This is a separate process, lets exit with the same exit code as cmd
    sys.exit(ec)


def _rs(length):
    return ''.join(
        random.choice(string.ascii_lowercase) for _ in range(length))


def _load_config():
    global config
    cfg = os.path.dirname(os.path.realpath(__file__)) + "/" + "config.yaml"
    with open(cfg, 'r') as array_data:
        config = yaml.safe_load(array_data.read())

    # If the user didn't specify a full path in the configuration file we
    # expect it in the same directory as this file
    if config['PROGRAM'][0] != '/':
        config['PROGRAM'] = os.path.dirname(os.path.realpath(__file__)) + \
                            '/' + config['PROGRAM']

    # Lets make sure import external files/directories are present
    if not os.path.exists(config['PROGRAM']):
        testlib.p("config PROGRAM %s does not exist" % config['PROGRAM'])
        sys.exit(1)

    if not (os.path.exists(config['LOGDIR']) and os.path.isdir(
            config['LOGDIR']) and os.access(config['LOGDIR'], os.W_OK)):
        testlib.p("config LOGDIR not preset or not a "
                  "directory %s or not writeable" % (config['LOGDIR']))
        sys.exit(1)


def _remove_file(job_id):
    try:
        os.remove(_file_name(job_id))
    except IOError as ioe:
        pass


def _update_state(job_id):
    global jobs
    job = jobs[job_id]

    # See if the process has ended
    p = job['PROCESS']<commit_msg>node.py: Add logging for _remove_file<commit_after>#!/usr/bin/env python2
"""
WARNING!  This file is auto updated from the node manager.  Any changes will
be lost when the client disconnects and reconnects.

This file is compatible with python2 and python3.

Theory of operation
1. Read the config file getting
   - Server and port to connect too
   - What arrays are available to use
2. Connect to the server
3. Wait for a command request
4. If server connection goes away or haven't received ping, close connection
   and try to periodically re-establish communication
Command requests include:
ping - check to see if the connection is working
arrays - Return array information on what is available to test
running - Return which arrays are currently running tests
job_create - Submit a new test to run (Creates a new process)
jobs - Return job state information on all submitted jobs
job - Return job state information about a specific job
job_delete - Delete the specific job request freeing resources
job_completion - Retrieve the exit code and log for specified job
Service which runs a test(s) on the same box as service
 See: https://github.com/tasleson/lsm-ci/blob/master/LICENSE
"""

import string
import random
import yaml
import os
import json
import sys
from subprocess import call
from multiprocessing import Process
import testlib
import time
import traceback
import tempfile
import shutil

jobs = {}
config = {}

STARTUP_CWD = ""

NODE = None


def _lcall(command, job_id):
    """
    Call an executable and return a tuple of exitcode, stdout&stderr
    """

    # Write output to a file so we can see what's going on while it's running
    f = '/tmp/%s.out' % job_id

    with open(f, 'w', buffering=1) as log:  # Max buffer 1 line (text mode)
        exit_value = call(command, stdout=log, stderr=log)
    return exit_value, f


def _file_name(job_id):
    # If this log directory is located in /tmp, the system may remove the
    # directory after a while, making us fail to log when needed.
    log_dir = config["LOGDIR"]
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    base = '%s/%s' % (log_dir, job_id)
    return base + ".out"


def _run_command(job_id, args):
    cmd = [config["PROGRAM"]]
    log_dir = config["LOGDIR"]

    cmd.extend(args)

    (ec, output_file) = _lcall(cmd, job_id)
    log = _file_name(job_id)

    # Read in output file in it's entirety
    with open(output_file, 'r') as o:
        out = o.read()

    # Delete file to prevent /tmp from filling up
    os.remove(output_file)

    with open(log, 'w') as error_file:
        error_file.write(yaml.dump(dict(EC=str(ec), OUTPUT=out)))
        error_file.flush()

    # This is a separate process, lets exit with the same exit code as cmd
    sys.exit(ec)


def _rs(length):
    return ''.join(
        random.choice(string.ascii_lowercase) for _ in range(length))


def _load_config():
    global config
    cfg = os.path.dirname(os.path.realpath(__file__)) + "/" + "config.yaml"
    with open(cfg, 'r') as array_data:
        config = yaml.safe_load(array_data.read())

    # If the user didn't specify a full path in the configuration file we
    # expect it in the same directory as this file
    if config['PROGRAM'][0] != '/':
        config['PROGRAM'] = os.path.dirname(os.path.realpath(__file__)) + \
                            '/' + config['PROGRAM']

    # Lets make sure import external files/directories are present
    if not os.path.exists(config['PROGRAM']):
        testlib.p("config PROGRAM %s does not exist" % config['PROGRAM'])
        sys.exit(1)

    if not (os.path.exists(config['LOGDIR']) and os.path.isdir(
            config['LOGDIR']) and os.access(config['LOGDIR'], os.W_OK)):
        testlib.p("config LOGDIR not preset or not a "
                  "directory %s or not writeable" % (config['LOGDIR']))
        sys.exit(1)


def _remove_file(job_id):
    fn = _file_name(job_id)
    try:
        testlib.p("Deleting file: %s" % fn)
        os.remove(fn)
    except IOError as ioe:
        testlib.p("Error deleting file: %s, reason: %s" % (fn, str(ioe)))
        pass


def _update_state(job_id):
    global jobs
    job = jobs[job_id]

    # See if the process has ended
    p = job['PROCESS']
{'fraction_non_alphanumeric': 0.09687875150060024, 'fraction_numerical': 0.0028811524609843936, 'mean_word_length': 3.114074074074074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2900355', 'n_tokens_mistral': 2613, 'n_tokens_neox': 2517, 'n_words': 955}