<commit_before># -*- coding: utf-8 -*-
#BEGIN_HEADER
import time
import subprocess
import os
import tempfile
import shutil
from pprint import pformat
from DataFileUtil.DataFileUtilClient import DataFileUtil
from DataFileUtil.baseclient import ServerError as DFUError
from kb_ea_utils.kb_ea_utilsClient import kb_ea_utils
from Workspace.WorkspaceClient import Workspace
from Workspace.baseclient import ServerError as WorkspaceError
from numbers import Number
import six
import uuid
import urllib2
from contextlib import closing
import ftplib
import re
import gzip
#END_HEADER


class ReadsUtils:
    '''
    Module Name:
    ReadsUtils

    Module Description:
    Utilities for handling reads files.
    '''

    ######## WARNING FOR GEVENT USERS ####### noqa
    # Since asynchronous IO can lead to methods - even the same method -
    # interrupting each other, you must be *very* careful when using global
    # state. A method could easily clobber the state set by another while
    # the latter method is running.
    ######################################### noqa
    VERSION = "0.3.1"
    GIT_URL = "https://github.com/Tianhao-Gu/ReadsUtils.git"
    GIT_COMMIT_HASH = "26056b45180356363b088c1ec33979dcc6be2df0"

    #BEGIN_CLASS_HEADER

    TRUE = 'true'
    FALSE = 'false'

    FASTA_JAR = '/opt/lib/FastaValidator-1.0.jar'
    FASTQ_EXE = 'fastQValidator'

    FASTA_EXT = ['.fa', '.fas', '.fasta', '.fna']
    FASTQ_EXT = ['.fq', '.fastq', '.fnq']

    COMPRESS_EXT = ['.gz', '.gzip', '.bz', '.bzip', '.bz2', '.bzip2']

    PARAM_IN_LIB = 'read_libraries'
    PARAM_IN_INTERLEAVED = 'interleaved'

    SINGLE_END_TYPE = 'SingleEndLibrary'
    PAIRED_END_TYPE = 'PairedEndLibrary'
    # one of these should be deprecated
    KBASE_FILE = 'KBaseFile'
    KBASE_ASSEMBLY = 'KBaseAssembly'
    MODULE_NAMES = [KBASE_FILE, KBASE_ASSEMBLY]
    TYPE_NAMES = [SINGLE_END_TYPE, PAIRED_END_TYPE]

    # staging file prefix
    STAGING_FILE_PREFIX = '/data/bulk/'

    def log(self, message, prefix_newline=False):
        print(('\n' if prefix_newline else '') +
              str(time.time()) + ': ' + message)

    def xor(self, a, b):
        return bool(a) != bool(b)

    def _add_field(self, obj, params, field):
        f = params.get(field)
        if f:
            obj[field] = f

    def _check_pos(self, num, name):
        if num is not None:
            if not isinstance(num, Number):
                raise ValueError(name + ' must be a number')
            if num <= 0:
                raise ValueError(name + ' must be > 0')

    def _proc_upload_reads_params(self, params):

        fwdid = params.get('fwd_id')
        fwdfile = params.get('fwd_file')
        download_type = params.get('download_type')
        fwdurl = params.get('fwd_file_url')
        fwdstaging = params.get('fwd_staging_file_name')

        fwdid, reads_source = (self._process_fwd_params(
            fwdid, fwdfile, fwdurl, fwdstaging, download_type))

        wsid = params.get('wsid')
        wsname = params.get('wsname')
        if not self.xor(wsid, wsname):
            raise ValueError(
                'Exactly one of the workspace ID or name must be provided')
        dfu = DataFileUtil(self.callback_url)
        if wsname:
            self.log('Translating workspace name to id')
            if not isinstance(wsname, six.string_types):
                raise ValueError('wsname must be a string')
            wsid = dfu.ws_name_to_id(wsname)
            self.log('translation done')
        del wsname
        objid = params.get('objid')
        name = params.get('name')
        if not self.xor(objid, name):
            raise ValueError(
                'Exactly one of the object ID or name must be provided')
        revid = params.get('rev_id')
        revfile = params.get('rev_file')
        revurl = params.get('rev_file_url')
        revstaging = params.get('rev_staging_file_name')

        self._check_rev_params(revid, revfile, revurl, revstaging, fwdfile,
                               fwdurl, fwdstaging, reads_source)

        if revurl:
            revid = revurl
        elif revstaging:
            revid = revstaging
        elif revfile:
            revid = os.path.abspath(os.path.expanduser(revfile))

        interleaved = 0
        kbtype = 'KBaseFile.SingleEndLibrary'
        single_end = True
        if params.get('interleaved') or revid:
            interleaved = 1
            kbtype = 'KBaseFile.PairedEndLibrary'
            single_end = False

        source_reads_ref = params.get('source_reads_ref')
        if source_reads_ref:
            o = self._propagate_reference_reads_info(params, dfu,
                                                     source_reads_ref,
                                                     interleaved, single_end)
        else:
            o = self._build_up_reads_data(params, single_end)
        return o, wsid, name, objid, kbtype, single_end, fwdid, revid, reads_source

    def _check_rev_params(self, revid, revfile, revurl, revstaging,
                          fwdfile, fwdurl, fwdstaging, reads_source):
        if sum(bool(e) for e in [revid, revfile, revurl, revstaging]) > 1:
            raise ValueError('Cannot specify more than one rev file source')

        if reads_source == 'shock' and any([revfile, revurl, revstaging]):
            raise ValueError('Cannot specify a local, web or staging reverse reads file ' +
                             'with a forward reads file in shock')
        if revid and reads_source != 'shock':
            raise ValueError('Cannot specify a reverse reads file in shock ' +
                             'with a local forward reads file')
        if revfile and not fwdfile:
            raise ValueError(
                'Specified local reverse file path but missing local forward file path')
        if revurl and not fwdurl:
            raise ValueError(
                'Specified reverse file URL but missing forward file URL')
        if revstaging and not fwdstaging:
            raise ValueError(
                'Specified reverse staging file but missing forward staging file')

    def _process_fwd_params(self, fwdid, fwdfile, fwdurl, fwdstaging, download_type):

        if sum(bool(e) for e in [fwdid, fwdfile, fwdurl, fwdstaging]) != 1:
            raise ValueError('Exactly one of a file, shock id, staging ' +
                             'file name or file url containing ' +
                             'a forwards reads file must be specified')
        if fwdurl:
            if not download_type:
                raise ValueError(
                    'Both download_type and fwd_file_url must be provided')
            reads_source = 'web'
            fwdid = fwdurl
        elif fwdstaging:
            reads_source = 'staging'
            fwdid = fwdstaging
        elif fwdid:
            reads_source = 'shock'
        else:
            reads_source = 'local'
            fwdid = os.path.abspath(os.path.expanduser(fwdfile))

        return fwdid, reads_source

    def _propagate_reference_reads_info(self, params, dfu, source_reads_ref,
                                        interleaved, single_end):
        # Means the uploaded reads is a result of an input reads object being filtered/trimmed
        # Make sure that no non file related parameters are set. If so throw
        # error.
        parameters_should_unfilled = ['insert_size_mean', 'insert_size_std_dev',
                                      'sequencing_tech', 'strain',
                                      'source', 'read_orientation_outward']
        if any(x in params for x in parameters_should_unfilled):
            self.log(("'source_reads_ref' was passed, making the following list of " +
                      "parameters {} erroneous to " +
                      "include").format(",".join(parameters_should_unfilled)))
            raise ValueError(("'source_reads_ref' was passed, making the following list of " +
                              "parameters : {} erroneous to " +
                              "include").format(", ".join(parameters_should_unfilled)))
        try:
            source_reads_object = dfu.get_objects({'object_refs':<commit_msg>add process_rev/fwd_params<commit_after># -*- coding: utf-8 -*-
#BEGIN_HEADER
import time
import subprocess
import os
import tempfile
import shutil
from pprint import pformat
from DataFileUtil.DataFileUtilClient import DataFileUtil
from DataFileUtil.baseclient import ServerError as DFUError
from kb_ea_utils.kb_ea_utilsClient import kb_ea_utils
from Workspace.WorkspaceClient import Workspace
from Workspace.baseclient import ServerError as WorkspaceError
from numbers import Number
import six
import uuid
import urllib2
from contextlib import closing
import ftplib
import re
import gzip
#END_HEADER


class ReadsUtils:
    '''
    Module Name:
    ReadsUtils

    Module Description:
    Utilities for handling reads files.
    '''

    ######## WARNING FOR GEVENT USERS ####### noqa
    # Since asynchronous IO can lead to methods - even the same method -
    # interrupting each other, you must be *very* careful when using global
    # state. A method could easily clobber the state set by another while
    # the latter method is running.
    ######################################### noqa
    VERSION = "0.3.1"
    GIT_URL = "https://github.com/Tianhao-Gu/ReadsUtils.git"
    GIT_COMMIT_HASH = "26056b45180356363b088c1ec33979dcc6be2df0"

    #BEGIN_CLASS_HEADER

    TRUE = 'true'
    FALSE = 'false'

    FASTA_JAR = '/opt/lib/FastaValidator-1.0.jar'
    FASTQ_EXE = 'fastQValidator'

    FASTA_EXT = ['.fa', '.fas', '.fasta', '.fna']
    FASTQ_EXT = ['.fq', '.fastq', '.fnq']

    COMPRESS_EXT = ['.gz', '.gzip', '.bz', '.bzip', '.bz2', '.bzip2']

    PARAM_IN_LIB = 'read_libraries'
    PARAM_IN_INTERLEAVED = 'interleaved'

    SINGLE_END_TYPE = 'SingleEndLibrary'
    PAIRED_END_TYPE = 'PairedEndLibrary'
    # one of these should be deprecated
    KBASE_FILE = 'KBaseFile'
    KBASE_ASSEMBLY = 'KBaseAssembly'
    MODULE_NAMES = [KBASE_FILE, KBASE_ASSEMBLY]
    TYPE_NAMES = [SINGLE_END_TYPE, PAIRED_END_TYPE]

    # staging file prefix
    STAGING_FILE_PREFIX = '/data/bulk/'

    def log(self, message, prefix_newline=False):
        print(('\n' if prefix_newline else '') +
              str(time.time()) + ': ' + message)

    def xor(self, a, b):
        return bool(a) != bool(b)

    def _add_field(self, obj, params, field):
        f = params.get(field)
        if f:
            obj[field] = f

    def _check_pos(self, num, name):
        if num is not None:
            if not isinstance(num, Number):
                raise ValueError(name + ' must be a number')
            if num <= 0:
                raise ValueError(name + ' must be > 0')

    def _proc_upload_reads_params(self, params):

        fwdsource, reads_source = (self._process_fwd_params(
            params.get('fwd_id'), params.get('fwd_file'), params.get('fwd_file_url'), 
            params.get('fwd_staging_file_name'), params.get('download_type')))

        wsid = params.get('wsid')
        wsname = params.get('wsname')
        if not self.xor(wsid, wsname):
            raise ValueError(
                'Exactly one of the workspace ID or name must be provided')
        dfu = DataFileUtil(self.callback_url)
        if wsname:
            self.log('Translating workspace name to id')
            if not isinstance(wsname, six.string_types):
                raise ValueError('wsname must be a string')
            wsid = dfu.ws_name_to_id(wsname)
            self.log('translation done')
        del wsname
        objid = params.get('objid')
        name = params.get('name')
        if not self.xor(objid, name):
            raise ValueError(
                'Exactly one of the object ID or name must be provided')

        revsource = self._process_rev_params(params.get('rev_id'), params.get('rev_file'), 
            params.get('rev_file_url'), params.get('rev_staging_file_name'), reads_source)

        interleaved = 0
        kbtype = 'KBaseFile.SingleEndLibrary'
        single_end = True
        if params.get('interleaved') or params.get('rev_id'):
            interleaved = 1
            kbtype = 'KBaseFile.PairedEndLibrary'
            single_end = False

        source_reads_ref = params.get('source_reads_ref')
        if source_reads_ref:
            o = self._propagate_reference_reads_info(params, dfu,
                                                     source_reads_ref,
                                                     interleaved, single_end)
        else:
            o = self._build_up_reads_data(params, single_end)
        return o, wsid, name, objid, kbtype, single_end, fwdsource, revsource, reads_source

    def _check_rev_params(self, revid, revfile, revurl, revstaging, reads_source):
        if sum(bool(e) for e in [revid, revfile, revurl, revstaging]) > 1:
            raise ValueError('Cannot specify more than one rev file source')

        if reads_source == 'shock' and any([revfile, revurl, revstaging]):
            raise ValueError('Cannot specify a local, web or staging reverse reads file ' +
                             'with a forward reads file in shock')
        if revid and reads_source != 'shock':
            raise ValueError('Cannot specify a reverse reads file in shock ' +
                             'with a local forward reads file')
        if revfile and reads_source != 'local':
            raise ValueError(
                'Specified local reverse file path but missing local forward file path')
        if revurl and reads_source != 'web':
            raise ValueError(
                'Specified reverse file URL but missing forward file URL')
        if revstaging and reads_source != 'staging':
            raise ValueError(
                'Specified reverse staging file but missing forward staging file')

    def _process_rev_params(self, revid, revfile, revurl, revstaging, reads_source):

        self._check_rev_params(revid, revfile, revurl, revstaging, reads_source)

        if revurl:
            revsource = revurl
        elif revstaging:
            revsource = revstaging
        elif revfile:
            revsource = os.path.abspath(os.path.expanduser(revfile))
        else:
            revsource = revid

        return revsource


    def _process_fwd_params(self, fwdid, fwdfile, fwdurl, fwdstaging, download_type):

        if sum(bool(e) for e in [fwdid, fwdfile, fwdurl, fwdstaging]) != 1:
            raise ValueError('Exactly one of a file, shock id, staging ' +
                             'file name or file url containing ' +
                             'a forwards reads file must be specified')
        if fwdurl:
            if not download_type:
                raise ValueError(
                    'Both download_type and fwd_file_url must be provided')
            reads_source = 'web'
            fwdsource = fwdurl
        elif fwdstaging:
            reads_source = 'staging'
            fwdsource = fwdstaging
        elif fwdid:
            reads_source = 'shock'
            fwdsource = fwdid
        else:
            reads_source = 'local'
            fwdsource = os.path.abspath(os.path.expanduser(fwdfile))

        return fwdsource, reads_source

    def _propagate_reference_reads_info(self, params, dfu, source_reads_ref,
                                        interleaved, single_end):
        # Means the uploaded reads is a result of an input reads object being filtered/trimmed
        # Make sure that no non file related parameters are set. If so throw
        # error.
        parameters_should_unfilled = ['insert_size_mean', 'insert_size_std_dev',
                                      'sequencing_tech', 'strain',
                                      'source', 'read_orientation_outward']
        if any(x in params for x in parameters_should_unfilled):
            self.log(("'source_reads_ref' was passed, making the following list of " +
                      "parameters {} erroneous to " +
                      "include").format(",".join(parameters_should_unfilled)))
            raise ValueError(("'source_reads_ref' was passed, making the following list of " +
                              "parameters : {} erroneous to " +
                              "include").format(", ".join(parameters_should_unfilled)))
        try:
            source_reads_object = dfu.get_objects({'object_refs':
{'fraction_non_alphanumeric': 0.09227585367338244, 'fraction_numerical': 0.005234646052711668, 'mean_word_length': 2.1742658423493046, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2954059', 'n_tokens_mistral': 4892, 'n_tokens_neox': 4660, 'n_words': 1334}