<jupyter_start><jupyter_text>Last update: Feb 24 2022 by <EMAIL><jupyter_code>#pip3 install pandas
import pandas as pd

from py2neo import Node
from py2neo import Relationship
from py2neo import Transaction
from py2neo.database import Graph
#Load ibm-blogs.csv

blogs_df = pd.read_csv("ibm-blogs.csv")
print("Number of rows in ibm-blogs file: ", blogs_df.shape[0])
blogs_df.head()
id_df = blogs_df.set_index(blogs_df["id"])
id_df.drop(['id'], axis=1, inplace=True)
id_df.head()
nodes = sorted(list(blogs_df.id.unique()) )
len(nodes)
print(nodes)
def _node(an_id:str) -> Node:
    return Node(":Article" , title=id_df.at[an_id,'title'],  article_id=str(an_id),
                link=id_df.at[an_id,'link'], date=id_df.at[an_id,'date']
                )

d_nodes = dict((key, _node(key)) for (key) in nodes)
print(d_nodes)
url = "bolt://localhost:7687"
db = "talk2"
pwd = <PASSWORD>}"

neog = Graph(url, db, auth=('neo4j', pwd))
tx = neog.begin()
neog.delete_all()
neog.commit(tx) 
def countNodes():
    query = "MATCH (n) RETURN COUNT(n)"
    result = neog.query(query)
    print(result)
def createNodes(d_nodes:dict):
    for node in d_nodes.values():
        tx.create(node)
    print ("Neo4j nodes created")
# Creates a triple series with item1 - item2 - item3 as input

def create_tpl(itemOne, itemTwo, itemThree):
    tpl = []
    tpl.append(itemOne)
    tpl.append(itemTwo)
    tpl.append(itemThree)
    return tpl
def get_unique(column):
    """
    Get unique elements from a Pandas DF column
    Arguments:
        column: a DF column
    Returns:
        A node list with sorted, 
        unique elements of the DF column
    """
    nodes = ()
    nodes = sorted( list(column.unique() ) )
    return nodes 
countNodes()
#Create article nodes
tx = neog.begin()
createNodes(d_nodes)
neog.commit(tx) 
countNodes()
c_df = pd.read_csv ('final-categories.csv')
print(c_df)
rows = []
for _, row in c_df.iterrows():
    #print(row['Label'])
    cats = row['Label'].split('/')
    cats = list(filter(None, cats))
    
    for c in cats:
        tpl = create_tpl(row['URL'], c, row['Score'])
        rows.append(tpl)

#print(rows)
cat_df = pd.DataFrame(rows)
print(cat_df.shape)
print(cat_df.head())
#create node list from a tuple
#tuple:ordered collection of objects

#category node list
c_nodes = get_unique(cat_df[1])
print(c_nodes)
#type(c_nodes)
#Create a dictionary of categorry ID nodes with p_nodes list
d_nodes =()
def _node(category:str) -> Node:
    return Node(":Category",  name=category , label = category
    )

d_nodes = dict((key, _node(key)) for (key) in c_nodes) 
print("Dictionary size: " + str(len(d_nodes)) )
print(d_nodes)
#Create category nodes
tx = neog.begin()
createNodes(d_nodes)
neog.commit(tx) 
countNodes()
#Create URL -has-a-> Category relationships
rows=[]

for _, row in c_df.iterrows():
    #print(row['URL'])
    #print(row['Label'])
    cats = row['Label'].split('/')
    cats = list(filter(None, cats))
    #print(cats)
    #print(cats[-1])
    tpl = create_tpl(row['URL'], 'HAS-A', cats[-1])
    rows.append(tpl)


print(rows)
has_df = pd.DataFrame(rows)
print(has_df.shape)
print(has_df.head())
has_df.rename(columns={0: 'URL', 1: 'Relationship', 2: 'Category'}, inplace=True )
print(has_df.head())
print(has_df.shape)
#If we need to remove duplicates


#category1_df = has_df.drop_duplicates(
# subset = ['URL', 'Relationship', 'Category'],
#  keep = 'last').reset_index(drop = True)
  
# print latest dataframe
#print(category1_df.head())
#print(category1_df.shape)
tx = neog.begin()

for _, row in has_df.iterrows():
    node = "MATCH(a:`:Article`) WHERE a.link='" + row[0] + "' return a"
    a_node  =  neog.evaluate(node)
    node = "MATCH(c:`:Category`) WHERE c.label='" + row[2] + "' return c"
    c_node  =  neog.evaluate(node)
    tx.create(Relationship(a_node, row[1], c_node))

print("URL-relationship-Category completed")

neog.commit(tx)
#Create category-has-a->category relationships
rows=[]

for _, row in c_df.iterrows():
    #print(row['URL'])
    #print(row['Label'])
    cats = row['Label'].split('/')
    cats = list(filter(None, cats))
    #print('\nrow: ', cats)
    #print(type(cats))
    for c in cats:
        if (len(cats) >= 2):
            tpl = create_tpl(cats[0], 'HAS-A', cats[1])
            rows.append(tpl)
            cats.pop(0)

ontology_df = pd.DataFrame(rows)
print(ontology_df.shape)
print(ontology_df.head())

ontology_df.rename(columns={0: 'Category', 1: 'Relationship', 2: 'Category'}, inplace=True )
print(ontology_df.head())
print(ontology_df.shape)
new_df = ontology_df.drop_duplicates(
  subset = ['Category', 'Relationship', 'Category'],
  keep = 'last').reset_index(drop = True)
  
# print latest dataframe
print(new_df.head())
print(new_df.shape)
tx = neog.begin()

for _, row in new_df.iterrows():
    node = "MATCH(c:`:Category`) WHERE c.label='" + row[0] + "' return c"
    a_node  =  neog.evaluate(node)
    node = "MATCH(c:`:Category`) WHERE c.label='" + row[2] + "' return c"
    b_node  =  neog.evaluate(node)
    tx.create(Relationship(a_node, row[1], b_node))

print("Category-relationship-Category completed")

neog.commit(tx)
countNodes()
#open GDSL playground
#pagerank
#lovain<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16244239631336405, 'fraction_numerical': 0.008640552995391706, 'mean_word_length': 4.132019704433498, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '337709', 'license': 'MIT', 'n_tokens_mistral': 1957, 'n_tokens_neox': 1911, 'n_words': 432, 'path': 'code/AI-Talk-v1.0.ipynb', 'repo_name': 'jiportilla/gds-intro', 'size': '5217'}