<commit_before><tool id="GetFastQCRawDataFiles" name="GetFastQCRawDataFiles" version="0.01">
<description>get file-path from FastQC rawdata files in specified history</description>

<command interpreter="python">

        GetFastQCRawDataFiles.py --script_path "$runMe" --interpreter "python" 
            --tool_name "GetFastQCRawDataFiles" --inp_apikey "$inp_apikey" --inp_hname "$inp_hname" --output_dir "./" --output_tab "$tab_file" 
</command>
<inputs>
  <param name="job_name" type="text" label="Supply a name for the outputs to remind you what they contain" value="GetFastQCRawDataFiles"/> 
  <param name="inp_apikey" size="50" type="text" label="Copy and Paste your API_KEY from Menu-bar[User]>[API keys]>[Current API key] " value="">
    <validator type="empty_field" message="No API_KEY is available for the input text form"/>
  </param>
  <param name="inp_hname" size="30" type="text" label="Supply your Current history name from top of history pain " value="">
    <validator type="empty_field" message="No Current hitory name is available for the input text form"/>
  </param>
</inputs>
<outputs>
 <data format="tabular" name="tab_file" label="${job_name} on current history"/>

</outputs>
<configfiles>
<configfile name="runMe">
import sys
import os
print "python :" + sys.version
import ConfigParser
import dateutil.tz
import bioblend
from bioblend.galaxy import GalaxyInstance
from bioblend.galaxy.histories import HistoryClient
from bioblend.galaxy.datasets import DatasetClient
import pandas

print  os.getcwd()
print u"GetFastQCRawDataFiles.py Started......"

inp = sys.argv[1]
inp_apikey = sys.argv[2]
inp_hname = sys.argv[3]
outp = sys.argv[4]

print u"inp: " + inp
print u"inp_apikey: " + inp_apikey
print u"inp_hname: " + inp_hname
print u"outp: " + outp

if len(inp_apikey.strip()) == 0:
    raise Exception, ' API_KEY is not entered.'

if len(inp_hname.strip()) == 0:
    raise Exception, 'Current History Name is not entered.'

output_name = ": RawData"
output_chk = "FastqMcf"

print output_name + " /file chk str:" + output_chk

def makeDir(dname):
    if os.path.exists(dname) is False:
        os.mkdir(dname)
        print '%s (dir) created.' % dname
    else:
        print '%s (dir) is already exists.' % dname

def findDatfilePath(dClient, h_list, hname):

    dset_list = [dset for dset in [dClient.show_dataset(hst['id']) for hst in h_list]
        if dset['state'] == "ok" and dset['peek'] is not None]
    dset_list = [dset for dset in dset_list if str(output_chk) in dset['peek']]
    dset_df = pandas.DataFrame(dset_list)
    dset_list = dset_df[["hid","name","id"]]
    print dset_list

    file_dir = '/galaxy-central/config/get_dataset_path_out'
    makeDir(file_dir)
    path_list = [dl_dset(dt_id, dClient, file_dir) for dt_id in dset_list["id"]]
    path_df = pandas.DataFrame(path_list)
    path_df.columns = ['id','file_name']
    print path_df

    merge_df = pandas.merge(dset_list, path_df, how='right', on='id', left_index=False, right_index=False)
    print merge_df

    return merge_df

def dl_dset(id, dClient, dir):
    print id
    file_path = dir + '/' + str(id)
    dClient.download_dataset(id, file_path, use_default_filename=False)
    ret_set = [id, file_path]
    return ret_set

def get_sample_name(path_list, hst_cont_all):
    tmp01 = [item.split(':')[0].split(' ')[-1] for item in path_list['name']]
    tmp02 = [item.split(' ')[-1] for item in tmp01]
    path_list['mcf_hid'] = tmp02
    tmp03 = [[item['name'] for item in hst_cont_all if item['hid'] == int(hid)] for hid in tmp02]
    
    pair_count = 0
    pair_count = [item[0] for item in tmp03 if "mates" in item[0]]
    if pair_count >0:
        tmp04_bef = [item[0].replace(' on data ','|').split('|')[1].replace(',','') for item in tmp03]
        tmp04 = [item.split(' ')[0] if 'reads' in item else item.split(' ')[2] for item in tmp04_bef]
    else:
        tmp04 = [item[0].replace(' on data ','|').split('|')[1].split(' ')[0] for item in tmp03]

    path_list['sample_hid'] = tmp04
    tmp05 = [[item['name'] for item in hst_cont_all if item['hid'] == int(hid)] for hid in tmp04]
    tmp06 = [item[0].split('.')[0] for item in tmp05]
    path_list['sample_name'] = tmp06
    path_list = path_list.drop_duplicates(cols='sample_name', take_last=True)
    return path_list

def dataout(data, filename):
    data.to_csv(filename, sep="\t")

def main():
    GALAXY_URL = 'http://localhost:80/'
    API_KEY = inp_apikey
    gInstance = GalaxyInstance(url=GALAXY_URL, key=API_KEY)
    hClient = HistoryClient(gInstance)
    dClient = DatasetClient(gInstance)

    hst = [x for x in hClient.get_histories() if x['name'].strip() == inp_hname.strip()][0]
    if len(hst['id']) == 0:
        raise Exception, inp_hname + ' is not found. please check your current history name.'
    print hst['id']
    hst_cont_all = hClient.show_history(hst['id'], deleted=False, contents=True, visible=True, details=True)
    hst_cont_ok = [x for x in hst_cont_all if str(output_name) in x['name'] and x['deleted'] == False and x['state'] == 'ok' ]<commit_msg>modefied pandas.DataFrame.drop_duplicates args for ver 0.18.0<commit_after><tool id="GetFastQCRawDataFiles" name="GetFastQCRawDataFiles" version="0.01">
<description>get file-path from FastQC rawdata files in specified history</description>

<command interpreter="python">

        GetFastQCRawDataFiles.py --script_path "$runMe" --interpreter "python" 
            --tool_name "GetFastQCRawDataFiles" --inp_apikey "$inp_apikey" --inp_hname "$inp_hname" --output_dir "./" --output_tab "$tab_file" 
</command>
<inputs>
  <param name="job_name" type="text" label="Supply a name for the outputs to remind you what they contain" value="GetFastQCRawDataFiles"/> 
  <param name="inp_apikey" size="50" type="text" label="Copy and Paste your API_KEY from Menu-bar[User]>[API keys]>[Current API key] " value="">
    <validator type="empty_field" message="No API_KEY is available for the input text form"/>
  </param>
  <param name="inp_hname" size="30" type="text" label="Supply your Current history name from top of history pain " value="">
    <validator type="empty_field" message="No Current hitory name is available for the input text form"/>
  </param>
</inputs>
<outputs>
 <data format="tabular" name="tab_file" label="${job_name} on current history"/>

</outputs>
<configfiles>
<configfile name="runMe">
import sys
import os
print "python :" + sys.version
import ConfigParser
import dateutil.tz
import bioblend
from bioblend.galaxy import GalaxyInstance
from bioblend.galaxy.histories import HistoryClient
from bioblend.galaxy.datasets import DatasetClient
import pandas

print  os.getcwd()
print u"GetFastQCRawDataFiles.py Started......"

inp = sys.argv[1]
inp_apikey = sys.argv[2]
inp_hname = sys.argv[3]
outp = sys.argv[4]

print u"inp: " + inp
print u"inp_apikey: " + inp_apikey
print u"inp_hname: " + inp_hname
print u"outp: " + outp

if len(inp_apikey.strip()) == 0:
    raise Exception, ' API_KEY is not entered.'

if len(inp_hname.strip()) == 0:
    raise Exception, 'Current History Name is not entered.'

output_name = ": RawData"
output_chk = "FastqMcf"

print output_name + " /file chk str:" + output_chk

def makeDir(dname):
    if os.path.exists(dname) is False:
        os.mkdir(dname)
        print '%s (dir) created.' % dname
    else:
        print '%s (dir) is already exists.' % dname

def findDatfilePath(dClient, h_list, hname):

    dset_list = [dset for dset in [dClient.show_dataset(hst['id']) for hst in h_list]
        if dset['state'] == "ok" and dset['peek'] is not None]
    dset_list = [dset for dset in dset_list if str(output_chk) in dset['peek']]
    dset_df = pandas.DataFrame(dset_list)
    dset_list = dset_df[["hid","name","id"]]
    print dset_list

    file_dir = '/galaxy-central/config/get_dataset_path_out'
    makeDir(file_dir)
    path_list = [dl_dset(dt_id, dClient, file_dir) for dt_id in dset_list["id"]]
    path_df = pandas.DataFrame(path_list)
    path_df.columns = ['id','file_name']
    print path_df

    merge_df = pandas.merge(dset_list, path_df, how='right', on='id', left_index=False, right_index=False)
    print merge_df

    return merge_df

def dl_dset(id, dClient, dir):
    print id
    file_path = dir + '/' + str(id)
    dClient.download_dataset(id, file_path, use_default_filename=False)
    ret_set = [id, file_path]
    return ret_set

def get_sample_name(path_list, hst_cont_all):
    tmp01 = [item.split(':')[0].split(' ')[-1] for item in path_list['name']]
    tmp02 = [item.split(' ')[-1] for item in tmp01]
    path_list['mcf_hid'] = tmp02
    tmp03 = [[item['name'] for item in hst_cont_all if item['hid'] == int(hid)] for hid in tmp02]
    
    pair_count = 0
    pair_count = [item[0] for item in tmp03 if "mates" in item[0]]
    if pair_count >0:
        tmp04_bef = [item[0].replace(' on data ','|').split('|')[1].replace(',','') for item in tmp03]
        tmp04 = [item.split(' ')[0] if 'reads' in item else item.split(' ')[2] for item in tmp04_bef]
    else:
        tmp04 = [item[0].replace(' on data ','|').split('|')[1].split(' ')[0] for item in tmp03]

    path_list['sample_hid'] = tmp04
    tmp05 = [[item['name'] for item in hst_cont_all if item['hid'] == int(hid)] for hid in tmp04]
    tmp06 = [item[0].split('.')[0] for item in tmp05]
    path_list['sample_name'] = tmp06
    if os.path.isfile("/galaxy-central/tool-data/sailfish_0.9_index.loc"):
        path_list = path_list.drop_duplicates(subset=['sample_name'], take_last=True)
    else:
        path_list = path_list.drop_duplicates(cols='sample_name', take_last=True)
    return path_list

def dataout(data, filename):
    data.to_csv(filename, sep="\t")

def main():
    GALAXY_URL = 'http://localhost:80/'
    API_KEY = inp_apikey
    gInstance = GalaxyInstance(url=GALAXY_URL, key=API_KEY)
    hClient = HistoryClient(gInstance)
    dClient = DatasetClient(gInstance)

    hst = [x for x in hClient.get_histories() if x['name'].strip() == inp_hname.strip()][0]
    if len(hst['id']) == 0:
        raise Exception, inp_hname + ' is not found. please check your current history name.'
    print hst['id']
    hst_cont_all = hClient.show_history(hst['id'], deleted=False, contents=True, visible=True, details=True)
    hst_cont_ok = [x for x in hst_cont_all if str(output_name) in x['name'] and x['deleted'] == False and x['state'] == 'ok' ]
{'fraction_non_alphanumeric': 0.14565826330532214, 'fraction_numerical': 0.01429537332174249, 'mean_word_length': 4.200401808136615, 'pattern_counts': {'":': 2, '<': 41, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6263384', 'n_tokens_mistral': 3811, 'n_tokens_neox': 3723, 'n_words': 963}