<commit_before>import scipy.stats as stats
import pandas as pd
import time
from modules.turtleGrapher.turtle_utils import generate_uri as gu
from modules.groupComparisons.sparql_queries import query, get_instances

def fishers(queryAttributeUriA, queryAttributeUriB, targetUri, results):
    # split the results into sub vars
    ## num of uniq subjects in A
    nA = results[0]['N']
    ## num of uniq subjects in B
    nB = results[1]['N']
    ## dictionary with the results from A
    dictA = results[0]['d']
    ## dictionary with the results from B
    dictB = results[1]['d']

    # join all possible targets as req for Fisher's
    # we could instead query the db for `DISTINCT ?s WHERE ?s a targetUri`, but I'm not interested in targets that neither queryA nor queryB has.
    all_targets = set(dictA.keys())
    all_targets.update(dictB.keys())

    # create a pandas dataframe for storing aggregate results from fisher's
    df = pd.DataFrame(columns=['target','queryA','queryB','presentQueryA','absentQueryA','presentQueryB','absentQueryB','pvalue','oddsratio'])

    # iterate through targets and perform fisher's
    for index, target in enumerate(all_targets):
        # tags for dataframe
        queryA = queryAttributeUriA
        queryB = queryAttributeUriB
        # check if target is found in queryA
        if target in dictA.keys():
            presentQueryA = len(dictA[target])
        else:
            presentQueryA = 0
        absentQueryA = nA - presentQueryA<commit_msg>FIX: list assignment out of range<commit_after>import logging
import scipy.stats as stats
import pandas as pd
import time
from modules.turtleGrapher.turtle_utils import generate_uri as gu
from modules.groupComparisons.sparql_queries import query, get_instances

# logging
log_file = initialize_logging()
log = logging.getLogger(__name__)

def fishers(queryAttributeUriA, queryAttributeUriB, targetUri, results):
    log.info(results)
    # split the results into sub vars
    ## num of uniq subjects in A
    nA = results[0]['N']
    ## num of uniq subjects in B
    nB = results[1]['N']
    ## dictionary with the results from A
    dictA = results[0]['d']
    ## dictionary with the results from B
    dictB = results[1]['d']

    # join all possible targets as req for Fisher's
    # we could instead query the db for `DISTINCT ?s WHERE ?s a targetUri`, but I'm not interested in targets that neither queryA nor queryB has.
    all_targets = set(dictA.keys())
    all_targets.update(dictB.keys())

    # create a pandas dataframe for storing aggregate results from fisher's
    df = pd.DataFrame(columns=['target','queryA','queryB','presentQueryA','absentQueryA','presentQueryB','absentQueryB','pvalue','oddsratio'])

    # iterate through targets and perform fisher's
    for index, target in enumerate(all_targets):
        # tags for dataframe
        queryA = queryAttributeUriA
        queryB = queryAttributeUriB
        # check if target is found in queryA
        if target in dictA.keys():
            presentQueryA = len(dictA[target])
        else:
            presentQueryA = 0
        absentQueryA = nA - presentQueryA
{'fraction_non_alphanumeric': 0.08806636885768986, 'fraction_numerical': 0.003190810465858328, 'mean_word_length': 3.5172910662824206, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '961377', 'n_tokens_mistral': 917, 'n_tokens_neox': 878, 'n_words': 310}