<commit_before>import datetime
import json
import time
from uuid import uuid4
import random

from wranglertools import fdnDCIC


def convert_param(parameter_dict, vals_as_string=False):
    '''
    converts dictionary format {argument_name: value, argument_name: value, ...}
    to {'workflow_argument_name': argument_name, 'value': value}
    '''
    metadata_parameters = []
    for k, v in parameter_dict.iteritems():
        # we need this to be a float or integer if it really is, else a string
        if not vals_as_string:
            try:
                v = float(v)
                if v % 1 == 0:
                    v = int(v)
            except ValueError:
                v = str(v)
        else:
            v = str(v)

        metadata_parameters.append({"workflow_argument_name": k, "value": v})

    return metadata_parameters


def create_ffmeta_awsem(workflow, app_name, input_files=None, parameters=None, title=None, uuid=None,
                        output_files=None, award='1U01CA200059-01', lab='4dn-dcic-lab',
                        run_status='started', run_platform='AWSEM', run_url='', **kwargs):

    input_files = [] if input_files is None else input_files
    parameters = [] if parameters is None else parameters

    if title is None:
        title = app_name + " run " + str(datetime.datetime.now())

    return WorkflowRunMetadata(workflow=workflow, app_name=app_name, input_files=input_files,
                               parameters=parameters, uuid=uuid, award=award,
                               lab=lab, run_platform=run_platform, run_url=run_url,
                               title=title, output_files=output_files, run_status=run_status)


def create_ffmeta(sbg, workflow, input_files=None, parameters=None, title=None, sbg_task_id=None,
                  sbg_mounted_volume_ids=None, sbg_import_ids=None, sbg_export_ids=None, uuid=None,
                  award='1U01CA200059-01', lab='4dn-dcic-lab', run_platform='SBG',
                  output_files=None, run_status='started', **kwargs):

    input_files = [] if input_files is None else input_files
    parameters = [] if parameters is None else parameters
    # TODO: this probably is not right
    sbg_export_ids = [] if sbg_export_ids is None else sbg_export_ids

    if title is None:
        title = sbg.app_name + " run " + str(datetime.datetime.now())

    if sbg_task_id is None:
        sbg_task_id = sbg.task_id

    if not sbg_mounted_volume_ids:
        try:
            sbg.volume_list[0]['name']
            sbg_mounted_volume_ids = [x['name'] for x in sbg.volume_list]
        except:
            sbg_mounted_volume_ids = [x for x in sbg.volume_list]

    if not sbg_import_ids:
        sbg_import_ids = sbg.import_id_list

    if not output_files:
        output_files = sbg.export_report
    else:
        # self.output_files may contain e.g. file_format and file_type information.
        for of in output_files:
            for of2 in sbg.export_report:
                if of['workflow_argument_name'] == of2['workflow_argument_name']:
                    for k, v in of2.iteritems():
                        of[k] = v

    return WorkflowRunMetadata(workflow, sbg.app_name, input_files, parameters,
                               sbg_task_id, sbg_import_ids, sbg_export_ids,
                               sbg_mounted_volume_ids, uuid,
                               award, lab, run_platform, title, output_files, run_status, **kwargs)


class WorkflowRunMetadata(object):
    '''
    fourfront metadata
    '''

    def __init__(self, workflow, app_name, input_files=[],
                 parameters=[], sbg_task_id=None,
                 sbg_import_ids=None, sbg_export_ids=None,
                 sbg_mounted_volume_ids=None, uuid=None,
                 award='1U01CA200059-01', lab='4dn-dcic-lab',
                 run_platform='SBG', title=None, output_files=None,
                 run_status='started', awsem_job_id=None,
                 run_url='', **kwargs):
        """Class for WorkflowRun that matches the 4DN Metadata schema
        Workflow (uuid of the workflow to run) has to be given.
        Workflow_run uuid is auto-generated when the object is created.
        """
        if run_platform == 'SBG':
            self.sbg_app_name = app_name
            if sbg_task_id is None:
                self.sbg_task_id = ''
            else:
                self.sbg_task_id = sbg_task_id
            if sbg_mounted_volume_ids is None:
                self.sbg_mounted_volume_ids = []
            else:
                self.sbg_mounted_volume_ids = sbg_mounted_volume_ids
            if sbg_import_ids is None:
                self.sbg_import_ids = []
            else:
                self.sbg_import_ids = sbg_import_ids
            if sbg_export_ids is None:
                self.sbg_export_ids = []
            else:
                self.sbg_export_ids = sbg_export_ids
        elif run_platform == 'AWSEM':
            self.awsem_app_name = app_name
            if awsem_job_id is None:
                self.awsem_job_id = ''<commit_msg>using app_name<commit_after>import datetime
import json
import time
from uuid import uuid4
import random

from wranglertools import fdnDCIC


def convert_param(parameter_dict, vals_as_string=False):
    '''
    converts dictionary format {argument_name: value, argument_name: value, ...}
    to {'workflow_argument_name': argument_name, 'value': value}
    '''
    metadata_parameters = []
    for k, v in parameter_dict.iteritems():
        # we need this to be a float or integer if it really is, else a string
        if not vals_as_string:
            try:
                v = float(v)
                if v % 1 == 0:
                    v = int(v)
            except ValueError:
                v = str(v)
        else:
            v = str(v)

        metadata_parameters.append({"workflow_argument_name": k, "value": v})

    return metadata_parameters


def create_ffmeta_awsem(workflow, app_name, input_files=None, parameters=None, title=None, uuid=None,
                        output_files=None, award='1U01CA200059-01', lab='4dn-dcic-lab',
                        run_status='started', run_platform='AWSEM', run_url='', **kwargs):

    input_files = [] if input_files is None else input_files
    parameters = [] if parameters is None else parameters

    if title is None:
        title = app_name + " run " + str(datetime.datetime.now())

    return WorkflowRunMetadata(workflow=workflow, app_name=app_name, input_files=input_files,
                               parameters=parameters, uuid=uuid, award=award,
                               lab=lab, run_platform=run_platform, run_url=run_url,
                               title=title, output_files=output_files, run_status=run_status)


def create_ffmeta(sbg, workflow, input_files=None, parameters=None, title=None, sbg_task_id=None,
                  sbg_mounted_volume_ids=None, sbg_import_ids=None, sbg_export_ids=None, uuid=None,
                  award='1U01CA200059-01', lab='4dn-dcic-lab', run_platform='SBG',
                  output_files=None, run_status='started', **kwargs):

    input_files = [] if input_files is None else input_files
    parameters = [] if parameters is None else parameters
    # TODO: this probably is not right
    sbg_export_ids = [] if sbg_export_ids is None else sbg_export_ids

    if title is None:
        title = sbg.app_name + " run " + str(datetime.datetime.now())

    if sbg_task_id is None:
        sbg_task_id = sbg.task_id

    if not sbg_mounted_volume_ids:
        try:
            sbg.volume_list[0]['name']
            sbg_mounted_volume_ids = [x['name'] for x in sbg.volume_list]
        except:
            sbg_mounted_volume_ids = [x for x in sbg.volume_list]

    if not sbg_import_ids:
        sbg_import_ids = sbg.import_id_list

    if not output_files:
        output_files = sbg.export_report
    else:
        # self.output_files may contain e.g. file_format and file_type information.
        for of in output_files:
            for of2 in sbg.export_report:
                if of['workflow_argument_name'] == of2['workflow_argument_name']:
                    for k, v in of2.iteritems():
                        of[k] = v

    return WorkflowRunMetadata(workflow, sbg.app_name, input_files, parameters,
                               sbg_task_id, sbg_import_ids, sbg_export_ids,
                               sbg_mounted_volume_ids, uuid,
                               award, lab, run_platform, title, output_files, run_status, **kwargs)


class WorkflowRunMetadata(object):
    '''
    fourfront metadata
    '''

    def __init__(self, workflow, app_name, input_files=[],
                 parameters=[], sbg_task_id=None,
                 sbg_import_ids=None, sbg_export_ids=None,
                 sbg_mounted_volume_ids=None, uuid=None,
                 award='1U01CA200059-01', lab='4dn-dcic-lab',
                 run_platform='SBG', title=None, output_files=None,
                 run_status='started', awsem_job_id=None,
                 run_url='', **kwargs):
        """Class for WorkflowRun that matches the 4DN Metadata schema
        Workflow (uuid of the workflow to run) has to be given.
        Workflow_run uuid is auto-generated when the object is created.
        """
        if run_platform == 'SBG':
            self.sbg_app_name = app_name
            self.app_name = app_name
            if sbg_task_id is None:
                self.sbg_task_id = ''
            else:
                self.sbg_task_id = sbg_task_id
            if sbg_mounted_volume_ids is None:
                self.sbg_mounted_volume_ids = []
            else:
                self.sbg_mounted_volume_ids = sbg_mounted_volume_ids
            if sbg_import_ids is None:
                self.sbg_import_ids = []
            else:
                self.sbg_import_ids = sbg_import_ids
            if sbg_export_ids is None:
                self.sbg_export_ids = []
            else:
                self.sbg_export_ids = sbg_export_ids
        elif run_platform == 'AWSEM':
            self.awsem_app_name = app_name
            self.app_name = app_name
            if awsem_job_id is None:
                self.awsem_job_id = ''
{'fraction_non_alphanumeric': 0.08449877750611247, 'fraction_numerical': 0.008606356968215158, 'mean_word_length': 2.0912938331318016, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4354100', 'n_tokens_mistral': 3178, 'n_tokens_neox': 3049, 'n_words': 719}