<commit_before># Perform the calculations for n-of-1 analysis

"""
import os.path, json, types,sys
import numpy as np

import newplacement
import compute_sparse_matrix
import leesL

def outputToJson(neighboorhood, xys, urls):
    '''
    This function takes the output from the newplacement call
     and puts it into the expected format
    @param neighboorhood: pandas df
    @param xys: pandas df
    @param urls: an array of URLs
    @return: dictionary to be turned into a JSON str
    '''
    #return dictionary to populate with results
    retDict = {"nodes":{}}

    #seperating the columns of the neighborhood df
    # for processing
    newNodes  = neighboorhood[neighboorhood.columns[0]]
    neighbors = neighboorhood[neighboorhood.columns[1]]
    scores    = neighboorhood[neighboorhood.columns[2]]
    #grab column names for indexing
    xcol = xys.columns[0]
    ycol = xys.columns[1]

    for i,node in enumerate(set(newNodes)):
        maskArr = np.array(newNodes == node)
        retDict['nodes'][node] = {}
        retDict['nodes'][node]['neighbors'] = dict(zip(neighbors.iloc[maskArr],scores.iloc[maskArr]))
        retDict['nodes'][node]['url'] = urls[i]
        retDict['nodes'][node]['x'] = xys.loc[node,xcol]
        retDict['nodes'][node]['y'] = xys.loc[node,ycol]

    return retDict

def tabArrayToPandas(tabArray):
    '''
    Takes a tab delemited array and makes a pandas dataframe
    @param tabArray:
    @return: pandas dataframe
    '''
    return pd.read_csv(StringIO('\n'.join(tabArray)),sep='\t',index_col=0)

def putDataIntoPythonStructs(featurePath,xyPath,tabSepArray):
    '''
    takes in the filenames and tab seperated array and puts in structures needed
     for placement calc
    @param featurePath:
    @param xyPath:
    @param tabSepArray:
    @return:
    '''
    return (compute_sparse_matrix.numpyToPandas(
                 *compute_sparse_matrix.read_tabular(featurePath)
                                                 ),
             leesL.readXYs(xyPath,preOrPost='pre'),
             tabArrayToPandas(tabSepArray)
             )

def entryPointFromWebApi(opts):

    if not hasattr(opts, 'top'):
        opts.top = 6

    # the files are good because the web handler has already checked that they
    # exist. lets get the python structs
    referenceDF, xyDF, newNodesDF =\
         putDataIntoPythonStructs(opts.fullFeatureMatrix,
                                  opts.xyPositions,
                                  opts.newNodes)
    #call the nOf1 function
    try:
        neighboorhood, xys, urls = newplacement.placeNew(newNodesDF,referenceDF,xyDF,opt.top,num_jobs=1)
        jdict = outputToJson(neighboorhood,xys,urls)
        return retDict
    except:
        return { 'error': 'Some error when calling newplacement.placeNew' }
"""
<commit_msg>switched input of new nodes, and return of entry point to python dict.<commit_after># Perform the calculations for n-of-1 analysis


import os.path, json, types,sys
import numpy as np
import pandas as pd
import newplacement
import compute_sparse_matrix
import leesL

def outputToDict(neighboorhood, xys, urls):
    '''
    This function takes the output from the newplacement call
      into the expected format
    @param neighboorhood: pandas df
    @param xys: pandas df
    @param urls: an array of URLs
    @return: dictionary to be turned into a JSON str
    '''
    #return dictionary to populate with results
    retDict = {"nodes":{}}

    #seperating the columns of the neighborhood df
    # for processing
    newNodes  = neighboorhood[neighboorhood.columns[0]]
    neighbors = neighboorhood[neighboorhood.columns[1]]
    scores    = neighboorhood[neighboorhood.columns[2]]
    #grab column names for indexing
    xcol = xys.columns[0]
    ycol = xys.columns[1]

    for i,node in enumerate(set(newNodes)):
        maskArr = np.array(newNodes == node)
        retDict['nodes'][node] = {}
        retDict['nodes'][node]['neighbors'] = dict(zip(neighbors.iloc[maskArr],scores.iloc[maskArr]))
        #add urls to the return struct
        #retDict['nodes'][node]['url'] = urls[i]
        retDict['nodes'][node]['x'] = xys.loc[node,xcol]
        retDict['nodes'][node]['y'] = xys.loc[node,ycol]

    return retDict

def nodesToPandas(pydict):
    '''
    function that takes the json['nodes'] stucture

    @param pydict: the dataIn['nodes'] structure,
                   currently a dict of dicts {columns -> {rows -> values}}
    @return: a pandas dataframe
    '''
    return pd.DataFrame(pydict)

def putDataIntoPythonStructs(featurePath,xyPath,nodesDict):
    '''
    takes in the filenames and tab seperated array and puts in structures needed
     for placement calc
    @param featurePath:
    @param xyPath:
    @param tabSepArray:
    @return:
    '''
    return (compute_sparse_matrix.numpyToPandas(
                 *compute_sparse_matrix.read_tabular(featurePath)
                                                 ),
             leesL.readXYs(xyPath,preOrPost='pre'),
             nodesToPandas(nodesDict)
             )

def entryPointFromWebApi(opts):

    if not hasattr(opts, 'top'):
        opts.top = 6

    # the files are good because the web handler has already checked that they
    # exist. lets get the python structs
    referenceDF, xyDF, newNodesDF =\
         putDataIntoPythonStructs(opts.fullFeatureMatrix,
                                  opts.xyPositions,
                                  opts.newNodes)
    #call the nOf1 function
    try:
        neighboorhood, xys, urls = newplacement.placeNew(newNodesDF,referenceDF,xyDF,opt.top,num_jobs=1)
        retDict = outputToDict(neighboorhood,xys,urls)
        return retDict
    except:
        return { 'error': 'Some error when calling newplacement.placeNew' }


{'fraction_non_alphanumeric': 0.09776391055642225, 'fraction_numerical': 0.0032934650719362107, 'mean_word_length': 2.781127129750983, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4577906', 'n_tokens_mistral': 1774, 'n_tokens_neox': 1726, 'n_words': 461}