<jupyter_start><jupyter_text>Case Study: CDS Methods Map CDS methods in Kitchen (2021b)<jupyter_code># Import dependencies
import requests
import pandas as pd
import numpy as np
import re
from datetime import date
import json
import os
import ast
# Import the knowledge base
kb_path = 'https://raw.githubusercontent.com/lustraka/data-analyst-portfolio-project-2022/main/data/'
kb_files = ['tao_iim.py', 'elem.txt', 'rels.csv', 'relt.csv']
for kb_file in kb_files:
  r = requests.get(kb_path+kb_file)
  with open(kb_file, 'wb') as file:
    file.write(r.content)

import tao_iim<jupyter_output><empty_output><jupyter_text>Update the Knowledge Base Programmatically<jupyter_code># Initialize & update dataframes
elem_df, rels_df, relt_df = tao_iim.load_elem_rel()
# Update here (applied updates below)<jupyter_output>Shapes of loaded files:  [(111, 3), (226, 3), (4, 2)]<jupyter_text>**2021-11-30** Updates:```pythonwith open('bbb', 'r') as file:  batches = ast.literal_eval(file.read())for batch in batches:  elem_df, rels_df, relt_df = tao_iim.add_batch(batch, elem_df, rels_df, relt_df)```where `bbb` stands for - `batches_20211129.txt` and - `batches_20211130.txt`.  Refactor concept names Refactor concept names in line with this concept model:![Ontology](https://www.plantuml.com/plantuml/png/FP2nJWCn38RtF8ML2Kxg6o3KjGDGiNIv9JcRdgMhKpcEKDyUNmWCrdy-V_fivLWjgRNR5fQa2F4aHYfay5wGPc6H2Ac2Pv1Y1ChNrGBx7oGn_c92o0y8IU1qXeIeL6iWGTZn8RrGXa-gfUdYpgPRTtgE-RdbZPTaN6IMU_uTUuxn6zbQS9Qd3xrUajBpB4M_E-GP0ej0VCclQwbMu-4GkSB-tK-BVOzNH-YM2pBzKQD5O8bzeHV4QLhOg4xJeBmRge4CrNqhZtzJxuPfl-f8WlwiFm00)<jupyter_code># Create a new dataframe with terms
renadf = elem_df[['term']].copy()

# Add new column with the most frequent prefix
renadf['new_term'] = renadf.term.apply(lambda t: 'pbl '+t)

# Set `term` as index for future transformation to dict
renadf.set_index('term', inplace=True)
renadf.tail()
# Adjust a new name for a web resource (there is only one)
inn_ele = elem_df.loc[elem_df.url.notna()].term.values[0]
renadf.at[inn_ele, 'new_term'] = 'inn Fraser 2019a'
renadf.at[inn_ele, 'new_term']<jupyter_output><empty_output><jupyter_text>**Ex-post Note**: Above should be `url Fraser 2019a`! Corrected directly in data ...<jupyter_code># Identify methods (they starts with a lowercase letter except the last element)
methods = elem_df.loc[elem_df.term.apply(lambda t: t[0].islower())]['term'].to_list()[:-1]
# Adjust prefixes for methods
for method in methods:
  print(method, end=' --> ')
  renadf.at[method, 'new_term'] = 'orw ' + method
  print(renadf.at[method, 'new_term'])
# Check the values visually
renadf.tail(12)
# Make a dict and rename terms
renadict = renadf.to_dict()['new_term']
elem_df.term = elem_df.term.apply(lambda t: renadict[t])
rels_df.elea = rels_df.elea.apply(lambda t: renadict[t])
rels_df.eleb = rels_df.eleb.apply(lambda t: renadict[t])
# Check elements
elem_df.head()
# Check relationships
rels_df.head()
# Check relationship types
rels_df.relt.value_counts()
renreldict = {'instantiates' : 'employs', 'uses' : 'employs', 'cites' : 'cites', 'embodies' : 'embodies'}
rels_df.relt = rels_df.relt.apply(lambda r: renreldict[r])
# Test relationships types
rels_df.relt.value_counts()<jupyter_output><empty_output><jupyter_text>Explore the Knowledge Base<jupyter_code>elem_df.loc[elem_df.title.isna()].sort_values(by='term')
for method in rels_df.query('elea == "orw critical data study"').eleb:
  print(method)
  print(rels_df.query(f'relt == "employs" and eleb ==\"{method}\"').elea.to_list())
# Select the relevant rows
rels_df.loc[(rels_df.relt == 'employs') & (rels_df.eleb == method) & (rels_df.elea.apply(lambda t: t[:3] == 'pbl'))]
# JOIN rels_df AND elem_df ON rels_df.elea = elem_df.term
df = rels_df.merge(elem_df[['term', 'title']], how='left', left_on='elea', right_on='term')

for method in df.query('elea == "orw critical data study"').eleb:
  print(f'- **{method[4:].title()}**')
  # Select the relevant rows
  publications = df.loc[(df.relt == 'employs') & (df.eleb == method) & (df.elea.apply(lambda t: t[:3] == 'pbl'))]  
  for _,row in publications.sort_values(by='elea').iterrows():
    print(f'\t- ({row[3]}) {row[4]}')
df.query(f'relt == "employs" and eleb ==\"{method}\"')<jupyter_output><empty_output><jupyter_text>Save the results<jupyter_code># Save the results
# do_it = False
do_it = True
if do_it:
  tao_iim.dump_elem_rel(elem_df, rels_df, relt_df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14841628959276018, 'fraction_numerical': 0.023076923076923078, 'mean_word_length': 6.430252100840336, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '124282', 'license': 'Unlicense', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1791, 'n_words': 360, 'path': 'cs01_cds_methods/20211130_Build_knowledge_base.ipynb', 'repo_name': 'lustraka/data-analyst-portfolio-project-2022', 'size': '4420'}