<commit_before>#!/usr/bin/env python

# standard library imports
import os
import sys
import traceback
import argparse
import json
import logging
import io
import re
import hashlib
import pprint

# 3rd party imports
import requests

# KBase imports
import biokbase.Transform.script_utils as script_utils


# conversion method that can be called if this module is imported
def convert(shock_service_url, handle_service_url, input_file_name, output_file_name, workspace_service_url, workspace_name, object_name, level=logging.INFO, logger=None):
    """
    Converts FASTA file to KBaseGenomes.ContigSet json string.  
    Note the MD5 for the contig is generated by uppercasing the sequence.
    The ContigSet MD5 is generated by taking the MD5 of joining the sorted list of individual contig's MD5s with a comma separator

    Args:
        shock_service_url: A url for the KBase SHOCK service.
        handle_service_url: A url for the KBase Handle Service.
        input_file_name: A file name for the input FASTA data.
        output_file_name: A file name where the output JSON string should be stored.
        workspace_service_url: A url for the Workspace Service.
        workspace_name: Name of the Workspace to store the ContigSet.
        object_name: Name of the ContigSet object to be saved in the Workspace.
        level: Logging level, defaults to logging.INFO.

    """

    if logger is None:
        logger = script_utils.getStderrLogger(__file__)
    
    logger.info("Starting conversion of FASTA to KBaseGenomes.ContigSet")
    token = os.environ.get('KB_AUTH_TOKEN')
    
    logger.info("Gathering information.")
 
    if not os.path.isfile(input_file_name):
        raise Exception("The input file name " + input_file_name + " does not exist")        

    contig_set_has_sequences = True 

    fasta_filesize = os.stat(input_file_name).st_size
    if fasta_filesize > 1000000000:
        # Fasta file too large to save sequences into the ContigSet object.        
        logger.info('Fasta input file %s is be too large, the contigset will not contain sequences'%(input_file_name)) 
#        print "Fasta input file %ifn is be too large, the contigset will not contain sequences"%(input_file_name)  
        contig_set_has_sequences = False 

    input_file_handle = open(input_file_name, 'r')
    fasta_header = None
    sequence_list = []
    fasta_dict = dict()
    first_header_found = False
    contig_set_md5_list = []
    pattern = re.compile(r'\s+')
    for current_line in input_file_handle:
        if (current_line[0] == ">"):
#        m = re.search(r"^>",current_line)
#        if m:
            #found a header line
            #Wrap up previous fasta sequence
            if len(sequence_list) == 0 and first_header_found:
                logger.info("There is no sequence related to fasta record : %s"%(fasta_header))        
                raise Exception("There is no sequence related to fasta record : " + fasta_header)    
            if not first_header_found:
                first_header_found = True
            else:
                #build up sequence and remove all white space
                total_sequence = ''.join(sequence_list)
                total_sequence = re.sub(pattern, '', total_sequence)
                fasta_key = fasta_header.strip()
                contig_dict = dict() 
                contig_dict["id"] = fasta_key 
                contig_dict["length"] = len(total_sequence) 
                contig_dict["name"] = fasta_key 
                contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" 
                contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() 
                contig_dict["md5"]= contig_md5 
                contig_set_md5_list.append(contig_md5) 
                if contig_set_has_sequences: 
                    contig_dict["sequence"]= total_sequence
                else: 
                    contig_dict["sequence"]= "" 
                fasta_dict[fasta_key] = contig_dict
               
                #get set up for next fasta sequence
                sequence_list = []
            fasta_header = current_line.replace('>','')
        else:
            sequence_list.append(current_line)
    #wrap up last fasta sequence
    if len(sequence_list) == 0 and first_header_found: 
        logger.info("There is no sequence related to fasta record : %s"%(fasta_header))        
        raise Exception("There is no sequence related to fasta record : " + fasta_header) 
    else: 
        #build up sequence and remove all white space                                                                                             
        total_sequence = ''.join(sequence_list)

        total_sequence = re.sub(pattern, '', total_sequence)
        fasta_key = fasta_header.strip()
        contig_dict = dict()
        contig_dict["id"] = fasta_key 
        contig_dict["length"] = len(total_sequence)
        contig_dict["name"] = fasta_key
        contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" 
        contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest()
        contig_dict["md5"]= contig_md5
        contig_set_md5_list.append(contig_md5)
        if contig_set_has_sequences: 
            contig_dict["sequence"]= total_sequence 
        else:
            contig_dict["sequence"]= "" 
        fasta_dict[fasta_key] = contig_dict 

    
    contig_set_dict = dict()
    contig_set_dict["md5"]=hashlib.md5(",".join(sorted(contig_set_md5_list))).hexdigest()
    contig_set_dict["id"]=output_file_name
    contig_set_dict["name"]=object_name
    contig_set_dict["source"]="KBase"
    contig_set_dict["source_id"]=os.path.basename(input_file_name) 
    contig_set_dict["contigs"]= [fasta_dict[x] for x in sorted(fasta_dict.keys())]

#    shock_response = script_utils.getShockID(logger, shock_service_url, input_file_name, token)
#    pprint.pprint(shock_response)
#   contig_set_dict["fasta_ref"]=?

    objectString = json.dumps(contig_set_dict, sort_keys=True, indent=4)
#    pprint.pprint(contig_set_dict)

    logger.info("Writing out JSON.")
    with open(output_file_name, "w") as outFile:
        outFile.write(objectString)
    
    logger.info("Conversion completed.")


# called only if script is run from command line
if __name__ == "__main__":	
    parser = argparse.ArgumentParser(prog='trns_transform_Fasta_to_ContigSet', 
                                     description='Converts FASTA file to ContigSet json string.',
                                     epilog='Authors: <NAME>, <NAME>')
    parser.add_argument('--shock_service_url', help='Shock url', action='store', type=str, default='https://kbase.us/services/shock-api/', nargs='?', required=True)
    parser.add_argument('--handle_service_url', help='Handle service url', action='store', type=str, default='https://kbase.us/services/handle_service/', nargs='?', required=True)
    parser.add_argument('--input_file_name', help ='Input Fasta file name', action='store', type=str, nargs='?', required=True)
    parser.add_argument('--output_file_name', help ='Output file name for the json representation of the ContigSet', action='store', type=str, nargs='?', required=False)
    parser.add_argument('--workspace_service_url', help='Workspace service url', action='store', type=str, default='https://kbase.us/services/ws', nargs='?', required=True)
    parser.add_argument('--workspace_name', help ='Workspace name to save to', action='store', type=str, nargs='?', required=True)
    parser.add_argument('--object_name', help ='Name to save the object as in the workspace', action='store', type=str, nargs='?', required=False)

#PERHAPS FORCE MAKE REFERENCE ONLY (NO SEQs in object) argument

    args = parser.parse_args()
    object_name = None
    if (args.object_name and len(args.object_name) == 1):
        object_name = args.object_name
    else:
        #default to input file name minus file extenstion
        base=os.path.basename(args.input_file_name)
        object_name = os.path.splitext(base)[0]

    output_file_name = None
    if (args.output_file_name and len(args.output_file_name) == 1):
        output_file_name = args.output_file_name
    else:
        #default to input file name minus file extenstion adding .json to the end
        base=os.path.basename(args.input_file_name)
        output_file_name = "%s_contig_set"%(os.path.splitext(base)[0])

    logger = script_utils.getStderrLogger(__file__)
    try:
        convert(args.shock_service_url, args.handle_service_url, args.input_file_name, output_file_name, args.workspace_service_url, args.workspace_name, object_name, logger=logger)
    except:
        logger.exception("".join(traceback.format_exc()))
        print "".join(traceback.format_exc())
        sys.exit(1)<commit_msg>adding more arguments faster check to see if no sequence contig<commit_after>#!/usr/bin/env python

# standard library imports
import os
import sys
import traceback
import argparse
import json
import logging
import io
import re
import hashlib
import pprint

# 3rd party imports
import requests

# KBase imports
import biokbase.Transform.script_utils as script_utils


# conversion method that can be called if this module is imported
def convert(shock_service_url, handle_service_url, input_file_name, output_file_name, working_directory, workspace_service_url, workspace_name, object_name, shock_id, level=logging.INFO, logger=None):
    """
    Converts FASTA file to KBaseGenomes.ContigSet json string.  
    Note the MD5 for the contig is generated by uppercasing the sequence.
    The ContigSet MD5 is generated by taking the MD5 of joining the sorted list of individual contig's MD5s with a comma separator

    Args:
        shock_service_url: A url for the KBase SHOCK service.
        handle_service_url: A url for the KBase Handle Service.
        input_file_name: A file name for the input FASTA data.
        output_file_name: A file name where the output JSON string should be stored.
        workspace_service_url: A url for the Workspace Service.
        workspace_name: Name of the Workspace to store the ContigSet.
        object_name: Name of the ContigSet object to be saved in the Workspace.
        level: Logging level, defaults to logging.INFO.

    """

    if logger is None:
        logger = script_utils.getStderrLogger(__file__)
    
    logger.info("Starting conversion of FASTA to KBaseGenomes.ContigSet")
    token = os.environ.get('KB_AUTH_TOKEN')
    
    logger.info("Gathering information.")
 
    if not os.path.isfile(input_file_name):
        raise Exception("The input file name " + input_file_name + " does not exist")        

    if not os.path.idir(args.working_directory):
        raise Exception("The working directory does not exist " + working_directory + " does not exist")        
    #CHECK PERMISSIONS?

    contig_set_has_sequences = True 

    fasta_filesize = os.stat(input_file_name).st_size
    if fasta_filesize > 1000000000:
        # Fasta file too large to save sequences into the ContigSet object.        
        logger.info('Fasta input file %s is be too large, the contigset will not contain sequences'%(input_file_name)) 
#        print "Fasta input file %ifn is be too large, the contigset will not contain sequences"%(input_file_name)  
        contig_set_has_sequences = False 

    input_file_handle = open(input_file_name, 'r')
    fasta_header = None
    sequence_list = []
    fasta_dict = dict()
    first_header_found = False
    contig_set_md5_list = []
    pattern = re.compile(r'\s+')
    sequence_exists = False
    for current_line in input_file_handle:
        if (current_line[0] == ">"):
            #found a header line
            #Wrap up previous fasta sequence
            if (not sequence_exists) and first_header_found:
                logger.info("There is no sequence related to fasta record : %s"%(fasta_header))        
                raise Exception("There is no sequence related to fasta record : " + fasta_header)    
            if not first_header_found:
                first_header_found = True
            else:
                #build up sequence and remove all white space
                total_sequence = ''.join(sequence_list)
                total_sequence = re.sub(pattern, '', total_sequence)
                fasta_key = fasta_header.strip()
                contig_dict = dict() 
                contig_dict["id"] = fasta_key 
                contig_dict["length"] = len(total_sequence) 
                contig_dict["name"] = fasta_key 
                contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" 
                contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest() 
                contig_dict["md5"]= contig_md5 
                contig_set_md5_list.append(contig_md5) 
                if contig_set_has_sequences: 
                    contig_dict["sequence"]= total_sequence
                else: 
                    contig_dict["sequence"]= "" 
                fasta_dict[fasta_key] = contig_dict
               
                #get set up for next fasta sequence
                sequence_list = []
                sequence_exists = False
            fasta_header = current_line.replace('>','')
        else:
            sequence_list.append(current_line)
            sequence_exists = True
    #wrap up last fasta sequence
    if (not sequence_exists) and first_header_found: 
        logger.info("There is no sequence related to fasta record : %s"%(fasta_header))        
        raise Exception("There is no sequence related to fasta record : " + fasta_header) 
    else: 
        #build up sequence and remove all white space                                                                                             
        total_sequence = ''.join(sequence_list)

        total_sequence = re.sub(pattern, '', total_sequence)
        fasta_key = fasta_header.strip()
        contig_dict = dict()
        contig_dict["id"] = fasta_key 
        contig_dict["length"] = len(total_sequence)
        contig_dict["name"] = fasta_key
        contig_dict["description"] = "Note MD5 is generated from uppercasing the sequence" 
        contig_md5 = hashlib.md5(total_sequence.upper()).hexdigest()
        contig_dict["md5"]= contig_md5
        contig_set_md5_list.append(contig_md5)
        if contig_set_has_sequences: 
            contig_dict["sequence"]= total_sequence 
        else:
            contig_dict["sequence"]= "" 
        fasta_dict[fasta_key] = contig_dict 
    
    contig_set_dict = dict()
    contig_set_dict["md5"]=hashlib.md5(",".join(sorted(contig_set_md5_list))).hexdigest()
    contig_set_dict["id"]=output_file_name
    contig_set_dict["name"]=object_name
    contig_set_dict["source"]="KBase"
    contig_set_dict["source_id"]=os.path.basename(input_file_name) 
    contig_set_dict["contigs"]= [fasta_dict[x] for x in sorted(fasta_dict.keys())]

    if shock_id is None:
        shock_response = script_utils.getShockID(logger, shock_service_url, input_file_name, token)
        pprint.pprint(shock_response)

#CHECK TO SEE WHAT IT IS IN THERE,  GET ID OUT?
#   contig_set_dict["fasta_ref"]=?

    objectString = json.dumps(contig_set_dict, sort_keys=True, indent=4)
#    pprint.pprint(contig_set_dict)

    logger.info("Writing out JSON.")
    with open(output_file_name, "w") as outFile:
        outFile.write(objectString)
    
    logger.info("Conversion completed.")


# called only if script is run from command line
if __name__ == "__main__":	
    parser = argparse.ArgumentParser(prog='trns_transform_Fasta_to_ContigSet', 
                                     description='Converts FASTA file to ContigSet json string.',
                                     epilog='Authors: <NAME>, <NAME>')
    parser.add_argument('--shock_service_url', help='Shock url', action='store', type=str, default='https://kbase.us/services/shock-api/', nargs='?', required=True)
    parser.add_argument('--handle_service_url', help='Handle service url', action='store', type=str, default='https://kbase.us/services/handle_service/', nargs='?', required=True)
    parser.add_argument('--input_file_name', help ='Input Fasta file name', action='store', type=str, nargs='?', required=True)
    parser.add_argument('--working_directory', help ='Directory the output file(s) should be written into', action='store', type=str, nargs='?', required=True)
    parser.add_argument('--output_file_name', help ='Output file name for the json representation of the ContigSet', action='store', type=str, nargs='?', required=False)
    parser.add_argument('--workspace_service_url', help='Workspace service url', action='store', type=str, default='https://kbase.us/services/ws', nargs='?', required=True)
    parser.add_argument('--workspace_name', help ='Workspace name to save to', action='store', type=str, nargs='?', required=True)
    parser.add_argument('--object_name', help ='Name to save the object as in the workspace', action='store', type=str, nargs='?', required=False)
    parser.add_argument('--shock_id', help='Shock node id if the fasta file already exists in shock', action='store', type=str, nargs='?', required=False)


#PERHAPS FORCE MAKE REFERENCE ONLY (NO SEQs in object) argument

    args = parser.parse_args()
    object_name = None
    if (args.object_name and len(args.object_name) == 1):
        object_name = args.object_name
    else:
        #default to input file name minus file extenstion
        base=os.path.basename(args.input_file_name)
        object_name = os.path.splitext(base)[0]

    output_file_name = None
    if (args.output_file_name and len(args.output_file_name) == 1):
        output_file_name = args.output_file_name
    else:
        #default to input file name minus file extenstion adding .json to the end
        base=os.path.basename(args.input_file_name)
        output_file_name = "%s_contig_set"%(os.path.splitext(base)[0])

    logger = script_utils.getStderrLogger(__file__)
    try:
        convert(args.shock_service_url, args.handle_service_url, args.input_file_name, output_file_name, args.working_directory, args.workspace_service_url, args.workspace_name, object_name, args.shock_id, logger=logger)
    except:
        logger.exception("".join(traceback.format_exc()))
        print "".join(traceback.format_exc())
        sys.exit(1)
{'fraction_non_alphanumeric': 0.09627210884353742, 'fraction_numerical': 0.004462585034013605, 'mean_word_length': 2.9475832438238454, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4032769', 'n_tokens_mistral': 5607, 'n_tokens_neox': 5367, 'n_words': 1384}