<commit_before>import pandas as pd
import numpy as np
import sys, os

from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.externals import joblib


model_path = sys.argv[1]
scaler_path = sys.argv[2]
working_dir = sys.argv[3]

dafs_path = working_dir + "/annotated_dafs.tsv"
mfe_path = working_dir + "/rna_fold_results.txt"
putative_stem_loops = working_dir + "/identified_mirnas.bed"

scaler = joblib.load(scaler_path)
model = joblib.load(model_path)

dafs = pd.read_csv(dafs_path, sep = "\t", index_col=3, names = list("ABCDEFGHIJKL"))
dafs.sort_index(inplace=True)

mfe = pd.read_csv(mfe_path, sep="\t", index_col=3, names=list("ABCDEFG"))

mfe.sort_index(inplace=True)
temp_df = pd.merge(dafs, pd.DataFrame(mfe['E']), left_index=True, right_index=True, how="inner")
temp_df.rename(columns={'E_x':'old_score', 'G':'mir_id', 'H':'blast_evalue', 'I':'pid', 'J':'cigar', 'K':'repeat_coverage', 
                   'L':'gc_perc', 'E_y':'mfe'}, inplace=True)

temp_df['size'] = temp_df.apply(lambda x: x['C'] - x['B'], axis = 1)
temp_df['norm_mfe'] = temp_df.apply(lambda x: x['mfe'] / x['size'], axis = 1)

df = temp_df[['size', 'blast_evalue', 'pid', 'repeat_coverage', 'gc_perc', 'mfe', 'norm_mfe']]
# scaled = scaler.fit_transform(df)
scaled = scaler.transform(df)
preds = model.predict(scaled)

df['predictions'] = preds

removed_dafs = df.query('predictions == 0')

removed_dafs.to_csv(working_dir + "/excluded_dafs.tsv", sep="\t")
df.to_csv(working_dir + "/labelled_dafs.tsv", sep="\t")

########################### db ids
putative_stem_loops_df = pd.read_csv(putative_stem_loops, sep="\t", index_col = 3, names=list("ABCDEFG"))
removed_dafs['coords'] = removed_dafs.index.astype(str)
putative_stem_loops_df['coords'] = putative_stem_loops_df.index.astype(str)

merged = pd.merge(putative_stem_loops_df, removed_dafs, left_on = 'coords', right_on = 'coords', how="inner")
merged.drop_duplicates(subset='coords',inplace=True)
merged['G'].to_csv(working_dir + "/mirnas_to_delete.txt", header=False, index=False)


<commit_msg>Bugfix: unsupported operand type(s) for /: 'str' and 'int'<commit_after>import pandas as pd
import numpy as np
import sys, os

from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.externals import joblib


model_path = sys.argv[1]
scaler_path = sys.argv[2]
working_dir = sys.argv[3]

dafs_path = working_dir + "/annotated_dafs.tsv"
mfe_path = working_dir + "/rna_fold_results.txt"
putative_stem_loops = working_dir + "/identified_mirnas.bed"

scaler = joblib.load(scaler_path)
model = joblib.load(model_path)

dafs = pd.read_csv(dafs_path, sep = "\t", index_col=3, names = list("ABCDEFGHIJKL"))
dafs.sort_index(inplace=True)

mfe = pd.read_csv(mfe_path, sep="\t", index_col=3, names=list("ABCDEFG"))

mfe.sort_index(inplace=True)
temp_df = pd.merge(dafs, pd.DataFrame(mfe['E']), left_index=True, right_index=True, how="inner")
temp_df.rename(columns={'E_x':'old_score', 'G':'mir_id', 'H':'blast_evalue', 'I':'pid', 'J':'cigar', 'K':'repeat_coverage', 
                   'L':'gc_perc', 'E_y':'mfe'}, inplace=True)

temp_df['size'] = temp_df.apply(lambda x: x['C'] - x['B'], axis = 1)
temp_df['norm_mfe'] = temp_df.apply(lambda x: float(x['mfe']) / x['size'], axis = 1)

df = temp_df[['size', 'blast_evalue', 'pid', 'repeat_coverage', 'gc_perc', 'mfe', 'norm_mfe']]
# scaled = scaler.fit_transform(df)
scaled = scaler.transform(df)
preds = model.predict(scaled)

df['predictions'] = preds

removed_dafs = df.query('predictions == 0')

removed_dafs.to_csv(working_dir + "/excluded_dafs.tsv", sep="\t")
df.to_csv(working_dir + "/labelled_dafs.tsv", sep="\t")

########################### db ids
putative_stem_loops_df = pd.read_csv(putative_stem_loops, sep="\t", index_col = 3, names=list("ABCDEFG"))
removed_dafs['coords'] = removed_dafs.index.astype(str)
putative_stem_loops_df['coords'] = putative_stem_loops_df.index.astype(str)

merged = pd.merge(putative_stem_loops_df, removed_dafs, left_on = 'coords', right_on = 'coords', how="inner")
merged.drop_duplicates(subset='coords',inplace=True)
merged['G'].to_csv(working_dir + "/mirnas_to_delete.txt", header=False, index=False)



{'fraction_non_alphanumeric': 0.18312854442344045, 'fraction_numerical': 0.004253308128544423, 'mean_word_length': 6.897388059701493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '115839', 'n_tokens_mistral': 1735, 'n_tokens_neox': 1656, 'n_words': 277}