<commit_before>#!/usr/bin/env python2.7
from pylab import *
import pandas
import sys


def main(args):
    # evidence_raw = pandas.read_table('evidence.txt')
    proteins_raw = pandas.read_table('proteinGroups.txt')
    phosphosites_raw = pandas.read_table('Phospho (STY)Sites.txt')

    # evidence = deduplicate(evidence_raw, "Protein Group IDs")
    # evidence = deduplicate(evidence, "Phospho (STY) Site IDs")
    phosphosites = deduplicate(phosphosites_raw, "Protein Group IDs")
    evi_norm = pandas.merge(phosphosites,
                            proteins_raw,
                            left_on="Protein Group IDs",
                            right_on="id",
                            how="left")
    evi_norm.to_csv('phospho_norm.txt', sep="\t")


def deduplicate(dfin, col):
    df = dfin
    drp = []
    new_entries = []
    for i in reversed(xrange(0, df.shape[0])):
        ids = df.iloc[i][col]
        if type(ids) is not str:
            continue
        tok = ids.split(";")
        if len(tok) > 1:
            for t in tok:
                new_entry = df.iloc[i]
                new_entry[col] = t
                new_entries.append(new_entry)
            drp.append(i)
    df = df.drop(df.index[drp])
    df = df.append(new_entries)
    return df


if __name__ == "__main__":
    sys.exit(main(sys.argv))<commit_msg>Renamed deduplicate to denormalize. Added some comments in denormalize indicating what needs to be fixed.<commit_after>#!/usr/bin/env python2.7
from pylab import *
import pandas
import sys


def main(args):
    # evidence_raw = pandas.read_table('evidence.txt')
    proteins_raw = pandas.read_table('proteinGroups.txt')
    phosphosites_raw = pandas.read_table('Phospho (STY)Sites.txt')

    # evidence = deduplicate(evidence_raw, "Protein Group IDs")
    # evidence = deduplicate(evidence, "Phospho (STY) Site IDs")
    phosphosites = denormalize(phosphosites_raw, "Protein Group IDs")
    evi_norm = pandas.merge(phosphosites,
                            proteins_raw,
                            left_on="Protein Group IDs",
                            right_on="id",
                            how="left")
    evi_norm.to_csv('phospho_norm.txt', sep="\t")


def denormalize(dfin, col):
    df = dfin
    drp = []
    new_entries = []
    for i in reversed(xrange(0, df.shape[0])):
        ids = df.iloc[i][col]
        if type(ids) is not str:
            continue
        tok = ids.split(";")
        if len(tok) > 1:
            idx = [] # needs indices of other columns with multiple values
            # for t in tok:
            for j in xrange(0, len(tok)):
                new_entry = df.iloc[i]
                new_entry[col] = tok[j]
                # for k in idx:
                    # set new_entry columns named in idx to df.iloc[i][k].split(";")[j]
                new_entries.append(new_entry)
            drp.append(i)
    df = df.drop(df.index[drp])
    df = df.append(new_entries)
    return df


if __name__ == "__main__":
    sys.exit(main(sys.argv))
{'fraction_non_alphanumeric': 0.11844724618447246, 'fraction_numerical': 0.0036496350364963502, 'mean_word_length': 2.067141403865717, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2577123', 'n_tokens_mistral': 1029, 'n_tokens_neox': 953, 'n_words': 217}