<jupyter_start><jupyter_text>from src.files.utils import trim_all<jupyter_code>def get_positive_amp_data(path):
    return pd.read_csv(path, sep=",", header=0).reset_index().drop('index', axis=1)[['Sequence', 'Activity']]

OTHER_ALPHABETS = "UOXBZJ"

def contains(other_alphabets, seq):
    for o in str(other_alphabets):
        if o in str(seq):
            return True
    return False

def trim_all(strings):
    return list(set(value.strip().strip(',').lower() for value in strings))

def sequence_filtering(data):
    sequences = data[data.apply(lambda r: not contains(OTHER_ALPHABETS, r['Sequence']), axis=1)]
    sequences = sequences[sequences.apply(lambda r: not str(r['Sequence']) == 'nan', axis=1)]
    sequences['Sequence'] = sequences['Sequence'].apply(lambda x: x.upper())
    sequences['Sequence'] = sequences['Sequence'].apply(lambda x: x.strip())
    return sequences

def massage_camp_data(data):
    sequences = sequence_filtering(data)
    sequences['Activity'] = sequences['Activity'].apply(lambda x: trim_all(str(x).split(',')))
    # sequences['Activity'] = sequences['Activity'].apply(lambda x: x.remove(''))
    sequences = sequences.reset_index().drop('index', axis=1)
    for i in range(sequences.__len__()):
        while '' in sequences.loc[i, 'Activity']:
            sequences.loc[i, 'Activity'].remove('')
    return sequences
avp_data = sequence_filtering(avp_data)
avp_data = avp_data.drop_duplicates(["Sequence"]).reset_index(drop=True)
avp_data.shape
tw_curated = get_positive_amp_data("../data/raw/positive_data_filtered2807.csv")
tw_curated = massage_camp_data(tw_curated).reset_index(drop=True)
tw_curated = tw_curated.drop_duplicates(["Sequence"]).reset_index(drop=True)
tw_curated.shape
mlb = preprocessing.MultiLabelBinarizer()
tw_curated_target = mlb.fit_transform(tw_curated.Activity)
tw_curated_target = pd.DataFrame(tw_curated_target, columns=mlb.classes_)
tw_curated_target
tw_curated_avp = tw_curated[tw_curated_target['antiviral'] == 1].reset_index(drop=True)
tw_curated_avp.shape
tw_curated_avp.to_csv("../data/raw/tw_curated_avp.csv", index=False)
avp_sequences = pd.concat([tw_curated_avp.Sequence, avp_data.Sequence], axis=0).reset_index(drop=True)
avp_sequences.shape
avp_sequences = avp_sequences.drop_duplicates().reset_index(drop=True)
avp_sequences.shape
avp_sequences.to_csv("../data/raw/avp_sequences.csv", index=False)
avppred_data_training1 = pd.read_excel("../data/raw/AVPpred_data.xlsx", sheet_name="Training1-AVP")
avppred_data_training1 = avppred_data_training1[["Sequence"]]
avppred_data_validation1 = pd.read_excel("../data/raw/AVPpred_data.xlsx", sheet_name="Validation1-AVP")
avppred_data_validation1 = avppred_data_validation1[["Sequence"]]
avppred_data = pd.concat([avppred_data_training1, avppred_data_validation1])
avppred_data = avppred_data.drop_duplicates(["Sequence"]).reset_index(drop=True)
avp_seq = pd.read_csv("../data/raw/avp_sequences.csv")
avp_seq_complete = pd.concat([avp_seq, avppred_data])
avp_seq_complete = avp_seq_complete.drop_duplicates(["Sequence"]).reset_index(drop=True)
avp_seq_complete.shape
avp_seq_complete = sequence_filtering(avp_seq_complete).reset_index(drop=True)
avp_seq_complete.to_csv("../data/raw/avp_sequences_tw_avpdb_avppred.csv", index=False)<jupyter_output><empty_output><jupyter_text>Non-AVP data<jupyter_code>avppred_data_training1 = pd.read_excel("../data/raw/AVPpred_data.xlsx", sheet_name="Training1-Non-AVP")
avppred_data_training1 = avppred_data_training1[["Sequence"]]
avppred_data_validation1 = pd.read_excel("../data/raw/AVPpred_data.xlsx", sheet_name="Validation1-Non-AVP")
avppred_data_validation1 = avppred_data_validation1[["Sequence"]]
avppred_data_training2 = pd.read_excel("../data/raw/AVPpred_data.xlsx", sheet_name="Training2-Non-AVP")
avppred_data_training2 = avppred_data_training1[["Sequence"]]
avppred_data_validation2 = pd.read_excel("../data/raw/AVPpred_data.xlsx", sheet_name="Validation2-Non-AVP")
avppred_data_validation2 = avppred_data_validation1[["Sequence"]]
non_avp_sequences = pd.concat([avppred_data_training1,avppred_data_validation1,
                               avppred_data_training2,avppred_data_validation2])
non_avp_sequences.shape
non_avp_sequences = sequence_filtering(non_avp_sequences)
non_avp_sequences = non_avp_sequences.drop_duplicates(["Sequence"]).reset_index(drop=True)
non_avp_sequences.shape
non_avp_sequences.to_csv("../data/raw/non_avp_sequences.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14007607965987917, 'fraction_numerical': 0.009174311926605505, 'mean_word_length': 7.646034816247583, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '258656', 'license': 'MIT', 'n_tokens_mistral': 1659, 'n_tokens_neox': 1620, 'n_words': 183, 'path': 'notebooks/data_curation.ipynb', 'repo_name': 'thoughtworks/antiviral-peptide-predictions-using-gan', 'size': '4469'}