<jupyter_start><jupyter_text>Spread of cards matched to term across different termsGO Term was reasonably consistent, everything else was much more varied<jupyter_code>#"split a multi choice answer, returning a series with a column 
# header and the responses with the header stripped out."
def count_unique_answers( a_series ):
    # variable to store the final response in as we build it
    tokenise = list()
    split_answer = pd.DataFrame(a_series.str.split("|", expand=True))
    for i in split_answer:   
        tokenise.extend(split_answer[i])
        
    response = {}
    for i in tokenise:
        if not pd.isnull(i):  
            if i in response:
                response[i] = response[i]+1
            else:
                response[i] = 1
    return response

result_counts = {}

for i in fm: 
    if (fm[i].dtype == np.object):
        result_counts[fm[i].name] = count_unique_answers(fm[i])
        
results = pd.DataFrame.from_dict(result_counts)
results

results = results.transpose()
results.head(10)
# GO Term is quite consistent in the matching
results["GO Term"]
# Symbol is the next best and is already a mess - it's used in a lot of different places.
results["symbol"]<jupyter_output><empty_output><jupyter_text>Variation within a given termWhen a term is matched to cards, how many different options are given in total, and per person?<jupyter_code>#"split a multi choice answer, returning a series with a column 
# header and the responses with the header stripped out."
def count_variation( a_series ):
    # variable to store the final response in as we build it
    split_answer = a_series.str.split("|")
    split_answer = split_answer.sort_values()
    
    term_combinations = list()
    for terms_list in split_answer:
        if isinstance(terms_list, list):
            #Record unique combinations of terms
            if terms_list not in term_combinations:
                term_combinations.append(terms_list)
    return term_combinations

#"split a multi choice answer, returning a series with a column 
# header and the responses with the header stripped out."
def count_unique_terms( a_series ):
    # variable to store the final response in as we build it
    split_answer = a_series.str.split("|")
    split_answer = split_answer.sort_values()
    
    unique_terms = list()
    for terms_list in split_answer:
        if isinstance(terms_list, list):
            #Record unique terms regardless of combo
            for individual_term in terms_list:
                if individual_term not in unique_terms:
                    unique_terms.append(individual_term)            

    return unique_terms

term_combinations = {}
unique_results_per_term = {}
for i in fm: 
    if (fm[i].dtype == np.object):
        
        #column name
        col = fm[i].name
        
        #calculate number of different term combos per matched field
        term_combos = count_variation(fm[i])
        term_combinations[col] = {"terms": term_combos, "count": len(term_combos)}
        
        #calculate number of different unique terms per field
        unique_terms = count_unique_terms(fm[i])
        unique_results_per_term[col] = {"terms": unique_terms, "count": len(unique_terms)}


print(pd.DataFrame(unique_results_per_term).transpose())
print(pd.DataFrame(term_combinations).transpose())<jupyter_output>terms  \
homo_sapiens.gff|100287102               [accession, identifier, gene, dataset/database...   
homo_sapiens.gff|HGNC:37102              [dataset/database, h. sapiens, gene, accession...   
homo_sapiens.gff|DDX11L1                 [Q9H4C3_HUMAN, gene, name, symbol, identifier,...   
flybase_d_melanogaster.gaf|FBgn0043467   [GO Term, accession, gene, identifier, name, s...   
flybase_d_melanogaster.gaf|GO:0048149    [Chromosome location, GO Term, accesssion, dat...   
ncbi_homo_sapiens.gene_info|9606         [h. sapiens, human, identifier, taxon id, orga...   
ncbi_homo_sapiens.gene_info|1            [chromosome, gene, identifier, protein, name, ...   
ncbi_homo_sapiens.gene_info|A1BG         [BRCA1_HUMAN, gene, name, symbol, organism, id...   
ncbi_homo_sapiens.gene_info|MIM:138670   [accession, chromosome, dataset/database, iden...   
ncbi_homo_sapiens.gene_info|HGNC:HGNC:5  [chromosome, databa[...]<jupyter_text>Entry pointsWe asked participants if they were thinking of doing research or an experiment themselves, which card would they consider the entry point to this set of items?<jupyter_code>#Data prep to load the data and subset for usage
entry_points = pd.read_csv("data/entry_point.csv")
 #subset the data to just the columns that answer this question
entry_terms = entry_points.iloc[:,1:2]
#let's take a peek at the data
entry_terms.head(3)
#"Split out responses to the entry point question and return a count of how many
# times each term is used"
def count_entry_point_unique_answers( a_series ):
    # variable to store the final response in as we build it
    tokenise = list()
    split_answer = pd.DataFrame(a_series.str.split("|", expand=True))
    for i in split_answer:   
        tokenise.extend(split_answer[i])
        
    response = {}
    for i in tokenise:
        if not pd.isnull(i):  
            if i in response:
                response[i] = response[i]+1
            else:
                response[i] = 1
    return response


#we'll store unique counts in this var
result_counts = {}

# split up the terms and count them
for i in entry_terms: 
    if (entry_terms[i].dtype == np.object):
        result_counts[entry_terms[i].name] = count_entry_point_unique_answers(entry_terms[i])

#show the results to the world as a nice lil table
print(pd.DataFrame(result_counts))<jupyter_output>term_selected
DOI                          1
accession                    2
chromosome                   1
d. melanogaster              1
database                     3
dataset                      1
disease                      2
ensembl                      1
flymine                      1
gene                         7
homologue                    1
identifier                   5
name                         1
organism                     2
pathway                      1
protein                      2
publication                  4
pubmed ID                    1
uniprot                      1<jupyter_text>Entry point typesLet's count the number of different response types (mostly information or bio centric)<jupyter_code>#subset the data 
entry_point_type = entry_points[["participant_id","bio_or_info"]]
#we'll store unique counts in this var
entry_point_type_result_counts = {}

# split up the terms and count them
for i in entry_point_type: 
    if (entry_point_type[i].dtype == np.object):
        entry_point_type_result_counts[entry_point_type[i].name] = count_entry_point_unique_answers(entry_point_type[i])
# Let's print each type as a nice table.
result_types = pd.DataFrame(entry_point_type_result_counts).transpose()
print(result_types)
bio_info = result_types[["bio","info", "both"]].transpose().sum()
other_or_unclear = result_types.transpose().sum() - bio_info

print(other_or_unclear)<jupyter_output>bio_or_info    3
dtype: int64
{'fraction_non_alphanumeric': 0.08076707726763718, 'fraction_numerical': 0.011058230683090706, 'mean_word_length': 2.7985114300903775, 'pattern_counts': {'":': 4, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '435565', 'license': 'MIT', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2099, 'n_words': 639, 'path': 'file_mapping_and_entry_points.ipynb', 'repo_name': 'yochannah/subjective-data-models-analysis', 'size': '7144'}