<commit_before>import hashlib
import signal
import subprocess
import uuid
import os

from datetime import datetime

#import chitin.client.api as api
from .api import base
from . import conf

def hashfile(path, start_clock, halg=hashlib.md5, bs=65536, force_hash=False):

    mod_time = os.path.getmtime(path)
    if mod_time >= int(start_clock.strftime("%s")) or force_hash:
        f = open(path, 'rb')
        buff = f.read(bs)
        halg = halg()
        halg.update(buff)
        while len(buff) > 0:
            buff = f.read(bs)
            halg.update(buff)
        f.close()
        return halg.hexdigest()
    else:
        # The file /probably/ hasn't change, so don't bother rehashing...
        return None

def parse_tokens(fields):
    dirs_l = []
    file_l = []
    for field_i, field in enumerate(fields):
        had_semicolon = False
        if field[-1] == ";":
            had_semicolon = True
            field = field.replace(";", "")

        #if field.startswith("chitin://"):
        #    resource = get_resource_by_uuid(field.split("chitin://")[1])
        #    if resource:
        #        field = resource["current_path"]
        abspath = os.path.abspath(field)

        # Does the path exist? We might want to add its parent directory
        if os.path.exists(abspath):
            field_ = abspath
            #if insert_uuids:
            #    resource = get_resource_by_path(field_)
            #    if resource:
            #        field_ = "chitin://" + str(resource["uuid"])

            if had_semicolon:
                fields[field_i] = field_ + ';' # Update the command to use the full abspath
            else:
                fields[field_i] = field_ # Update the command to use the full abspath
        else:
            continue

        ### Files
        if os.path.isfile(abspath):
            file_l.append(abspath)

        ### Dirs
        elif os.path.isdir(abspath):
            dirs_l.append(abspath)

            for item in os.listdir(abspath):
                i_abspath = os.path.join(abspath, item)
                if os.path.isdir(i_abspath):
                    dirs_l.append(i_abspath)
                else:
                    #TODO Do we want to keep track of the files of subfolders?
                    pass
    return {
        "fields": fields,
        "files": set(file_l),
        "dirs": set(dirs_l),
    }

def inflate_path_set(path_set):
    paths = set({})
    for item in path_set:
        item = os.path.abspath(item)

        if os.path.isdir(item):
            for subitem in os.listdir(item):
                i_abspath = os.path.join(item, subitem)
                if os.path.isdir(i_abspath):
                    pass
                else:
                    #TODO Do we want to keep track of the files of untargeted subfolders?
                    paths.add(i_abspath)
        elif os.path.isfile(item):
            paths.add(item)

    return paths

class ClientDaemon(object):
    @staticmethod
    def run_command(cmd_uuid, cmd_str):
        def preexec_function():
            # http://stackoverflow.com/questions/5045771/python-how-to-prevent-subprocesses-from-receiving-ctrl-c-control-c-sigint <3
            # Ignore the SIGINT signal by setting the handler to the standard signal handler SIG_IGN
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        #Check whether files have been altered outside of environment before proceeding
        #token_p = parse_tokens(cmd_str.split(" "))
        #for failed in check_integrity_set2(token_p["dirs"] | token_p["files"]):
        #        print("[WARN] '%s' has been modified outside of lab book." % failed)

        # Organise watch lists (to keep track of deleted files later)
        fields = cmd_str.split(" ")
        token_p = parse_tokens(fields)
        watched_dirs = token_p["dirs"]
        watched_files = token_p["files"]

        start_clock = datetime.now()
        proc = subprocess.Popen(
                cmd_str,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                #env=dict(os.environ).update(block["env_vars"]),
                preexec_fn = preexec_function,
        )
        stdout, stderr = proc.communicate()
        end_clock = datetime.now()
        return_code = proc.returncode

        run_meta = {"wall": str(end_clock - start_clock)}

        if return_code != 0:
            #TODO Future: We should do something here - like warn/stop the command?
            pass

        #####################################

        # Update field tokens (to find newly created files)
        fields = cmd_str.split(" ")
        token_p = parse_tokens(fields)
        watched_dirs = watched_dirs.union(token_p["dirs"])
        #watched_dirs.add(command_r["job_path"])
        watched_files = watched_files.union(token_p["files"])
        cmd_str = " ".join(token_p["fields"]) # Replace cmd_str to use abspaths

        # Parse the output
        meta = {}
        #if cmd.can_parse(fields[0]):
        #    parsed_meta = cmd.attempt_parse(fields[0], cmd_str, stdout, stderr)
        #    meta.update(parsed_meta)
        meta["run"] = run_meta

        # Look for changes
        paths = inflate_path_set(watched_dirs | watched_files).union(watched_files) # Forcibly add old watched files to find deletions
        resource_info = []
        for path in paths:
            resource_hash = '0'
            resource_size = 0
            resource_exists = os.path.exists(path)
            if resource_exists:
                resource_hash = hashfile(path, start_clock, force_hash=False)
                resource_size = os.path.getsize(path)

            resource_info.append({
                #TODO Need a nice way to get the NODE UUID
                "node_uuid": conf.NODE_UUID,
                "path": path,
                "exists": resource_exists,
                "hash": resource_hash,
                "size": resource_size,
            })

        # Terrible way to run filetype handlers
        #check_integrity_set2(watched_files)

        # Pretty hacky way to get the UUID cmd str
        #token_p = parse_tokens(fields, insert_uuids=True)
        #uuid_cmd_str = " ".join(token_p["fields"]) # Replace cmd_str to use abspaths
        base.emit2("command/update", {
            "cmd_uuid": cmd_uuid,
            "meta": meta,
            "return_code": return_code,
            "text": {
                "stdout": stdout.decode('utf-8'),
                "stderr": stderr.decode('utf-8'),
            },
            "resources": resource_info,
            "started_at": int(start_clock.strftime("%s")),
            "finished_at": int(end_clock.strftime("%s")),
        }, to_uuid=None)

class Client(object):

    def __init__(self):
        self.meta = {}

    def signal_handler(self):<commit_msg>Identify Resources that existed before Command execution<commit_after>import hashlib
import signal
import subprocess
import uuid
import os

from datetime import datetime

#import chitin.client.api as api
from .api import base
from . import conf

def hashfile(path, start_clock, halg=hashlib.md5, bs=65536, force_hash=False):

    mod_time = os.path.getmtime(path)
    if mod_time >= int(start_clock.strftime("%s")) or force_hash:
        f = open(path, 'rb')
        buff = f.read(bs)
        halg = halg()
        halg.update(buff)
        while len(buff) > 0:
            buff = f.read(bs)
            halg.update(buff)
        f.close()
        return halg.hexdigest()
    else:
        # The file /probably/ hasn't change, so don't bother rehashing...
        return None

def parse_tokens(fields):
    dirs_l = []
    file_l = []
    for field_i, field in enumerate(fields):
        had_semicolon = False
        if field[-1] == ";":
            had_semicolon = True
            field = field.replace(";", "")

        #if field.startswith("chitin://"):
        #    resource = get_resource_by_uuid(field.split("chitin://")[1])
        #    if resource:
        #        field = resource["current_path"]
        abspath = os.path.abspath(field)

        # Does the path exist? We might want to add its parent directory
        if os.path.exists(abspath):
            field_ = abspath
            #if insert_uuids:
            #    resource = get_resource_by_path(field_)
            #    if resource:
            #        field_ = "chitin://" + str(resource["uuid"])

            if had_semicolon:
                fields[field_i] = field_ + ';' # Update the command to use the full abspath
            else:
                fields[field_i] = field_ # Update the command to use the full abspath
        else:
            continue

        ### Files
        if os.path.isfile(abspath):
            file_l.append(abspath)

        ### Dirs
        elif os.path.isdir(abspath):
            dirs_l.append(abspath)

            for item in os.listdir(abspath):
                i_abspath = os.path.join(abspath, item)
                if os.path.isdir(i_abspath):
                    dirs_l.append(i_abspath)
                else:
                    #TODO Do we want to keep track of the files of subfolders?
                    pass
    return {
        "fields": fields,
        "files": set(file_l),
        "dirs": set(dirs_l),
    }

def inflate_path_set(path_set):
    paths = set({})
    for item in path_set:
        item = os.path.abspath(item)

        if os.path.isdir(item):
            for subitem in os.listdir(item):
                i_abspath = os.path.join(item, subitem)
                if os.path.isdir(i_abspath):
                    pass
                else:
                    #TODO Do we want to keep track of the files of untargeted subfolders?
                    paths.add(i_abspath)
        elif os.path.isfile(item):
            paths.add(item)

    return paths

class ClientDaemon(object):
    @staticmethod
    def run_command(cmd_uuid, cmd_str):
        def preexec_function():
            # http://stackoverflow.com/questions/5045771/python-how-to-prevent-subprocesses-from-receiving-ctrl-c-control-c-sigint <3
            # Ignore the SIGINT signal by setting the handler to the standard signal handler SIG_IGN
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        #Check whether files have been altered outside of environment before proceeding
        #token_p = parse_tokens(cmd_str.split(" "))
        #for failed in check_integrity_set2(token_p["dirs"] | token_p["files"]):
        #        print("[WARN] '%s' has been modified outside of lab book." % failed)

        # Organise watch lists (to keep track of deleted files later)
        fields = cmd_str.split(" ")
        token_p = parse_tokens(fields)
        watched_dirs = token_p["dirs"]
        watched_files = token_p["files"]

        precommand_paths = inflate_path_set( set(watched_files) )

        start_clock = datetime.now()
        proc = subprocess.Popen(
                cmd_str,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                #env=dict(os.environ).update(block["env_vars"]),
                preexec_fn = preexec_function,
        )
        stdout, stderr = proc.communicate()
        end_clock = datetime.now()
        return_code = proc.returncode

        run_meta = {"wall": str(end_clock - start_clock)}

        if return_code != 0:
            #TODO Future: We should do something here - like warn/stop the command?
            pass

        #####################################

        # Update field tokens (to find newly created files)
        fields = cmd_str.split(" ")
        token_p = parse_tokens(fields)
        watched_dirs = watched_dirs.union(token_p["dirs"])
        #watched_dirs.add(command_r["job_path"])
        watched_files = watched_files.union(token_p["files"])
        cmd_str = " ".join(token_p["fields"]) # Replace cmd_str to use abspaths

        # Parse the output
        meta = {}
        #if cmd.can_parse(fields[0]):
        #    parsed_meta = cmd.attempt_parse(fields[0], cmd_str, stdout, stderr)
        #    meta.update(parsed_meta)
        meta["run"] = run_meta

        # Look for changes
        paths = inflate_path_set(watched_dirs | watched_files)
        resource_info = []
        for path in paths:
            resource_hash = '0'
            resource_size = 0
            resource_exists = os.path.exists(path)
            if resource_exists:
                resource_hash = hashfile(path, start_clock, force_hash=path in precommand_paths)
                resource_size = os.path.getsize(path)

            resource_info.append({
                #TODO Need a nice way to get the NODE UUID
                "node_uuid": conf.NODE_UUID,
                "path": path,
                "exists": resource_exists,
                "precommand_exists": path in precommand_paths,
                "hash": resource_hash,
                "size": resource_size,
            })

        # Terrible way to run filetype handlers
        #check_integrity_set2(watched_files)

        # Pretty hacky way to get the UUID cmd str
        #token_p = parse_tokens(fields, insert_uuids=True)
        #uuid_cmd_str = " ".join(token_p["fields"]) # Replace cmd_str to use abspaths
        base.emit2("command/update", {
            "cmd_uuid": cmd_uuid,
            "meta": meta,
            "return_code": return_code,
            "text": {
                "stdout": stdout.decode('utf-8'),
                "stderr": stderr.decode('utf-8'),
            },
            "resources": resource_info,
            "started_at": int(start_clock.strftime("%s")),
            "finished_at": int(end_clock.strftime("%s")),
        }, to_uuid=None)

class Client(object):

    def __init__(self):
        self.meta = {}

    def signal_handler(self):
{'fraction_non_alphanumeric': 0.10844682700666473, 'fraction_numerical': 0.003911909591422776, 'mean_word_length': 1.9093782929399368, 'pattern_counts': {'":': 39, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4893775', 'n_tokens_mistral': 4229, 'n_tokens_neox': 4052, 'n_words': 962}