<jupyter_start><jupyter_text>Knowledge📚 Represent personal Knowledge as a graph, with nodes for articles and edges for connections between them. The connections are computed based on the number of tags in commun. This should serve as a way to discover visually new content, and browse through existing content.<jupyter_code>import pandas as pd
df = pd.read_csv('diigo/diigo_csv_2021_12_05.csv')
df.head()
df.tags.fillna('')
n = len(df)
n<jupyter_output><empty_output><jupyter_text>Tagslist<jupyter_code>list0 = ["," for i in range(len(df))]
# df0 = pd.DataFrame(list0)
s0 = pd.Series(list0)
s = df.tags.fillna('') + s0
tagsum = s.sum()
tagslist = tagsum.split(",")
tagslist = list(dict.fromkeys(tagslist))
tagslist.remove('')
p = len(tagslist)
p
tagslist[0:10]<jupyter_output><empty_output><jupyter_text>Handle multiple tags for one article<jupyter_code>df['tags'][4]
tagslist.index(df['tags'][4])
l = df['tags'][4].split(",")
l
l = df['tags'][2].split(",")
l
for i in l:
    print(tagslist.index(i))
df['tags'][2].split(",")[0]<jupyter_output><empty_output><jupyter_text>Handle no tag for one article<jupyter_code>i = 1054
# handle cases with no tag (nan)
df['tags'][i]
pd.isna(df['tags'][i])
from math import isnan
isnan(df['tags'][i])
df['tags'][[i]].isnull().sum()
df['tags'].isnull()
~pd.isna(df['tags'][i])<jupyter_output><empty_output><jupyter_text>Adjacency matrix linking articles and tags<jupyter_code>from IPython.display import Image 
Image(filename='img/file_1_vector.png',width=300)
# Create a vector of n tags appearing in the set of articles
# Incrementing in parallel the matrix T joining the articles (rows) with the corresponding tags(columns)

import numpy as np

T = np.zeros((n,p))

for index, row in df.iterrows():
    # iterate on the dataframe of articles 
    if not pd.isna(row['tags']):
        l = row['tags'].split(",")
        for i in l:
            j = tagslist.index(i)
            T[index,j] = 1
T[1,0] # article on "Topic Modeling" with tag "data science"
w = np.where(T[1]==1)
v = w[0]
v # array of positions of 1s in the vector T[1]
np.where(T[2]==1)
np.where(T[3]==1)
v = np.where(T==1)
v1 = v[0]
v2 = v[1]
v1,v2<jupyter_output><empty_output><jupyter_text>https://www.w3schools.com/python/numpy_array_search.asp<jupyter_code>T2 = np.transpose(T)
# Computation of the matrix Z which contains 
# on the diagonale the number of appearance di of each tag i
# on the other elements the degree d(i,j) of closeness of 2 tags (i,j)
# this matrix is symmetrical

Z = np.matmul(T2,T)
from IPython.display import Image 
Image(filename='img/adjacency_matrix_building.png')
Zdiag=[]
for i in range(len(Z)):
    Zdiag.append(Z[i,i])<jupyter_output><empty_output><jupyter_text>Sort the 100 most used tags<jupyter_code># pd.Series(tagslist)
tags = pd.DataFrame(tagslist, columns=["tag"])
tags["r"]=Zdiag
tags.head()
# zdiag = pd.DataFrame(Zdiag,columns = ["r"])
# Zdiag.sort(reverse=True)
# Zdiag
# Sorting by descending value of r, and selecting the 100 first tags
tags100 = tags.sort_values("r",ascending = False)[0:100]
tags100<jupyter_output><empty_output><jupyter_text>Edges list<jupyter_code>E = []
# Iterate on the upper triangle matrix above the diag of Z
for i in range(len(Z)):
    for j in range(i+1,len(Z)):
        if Z[i,j] !=0:
            E.append([i,j])
edges = pd.DataFrame(E, columns=["source","target"])
edges<jupyter_output><empty_output><jupyter_text>Nodes list<jupyter_code>nodes = pd.DataFrame(tagslist, columns=["tag"])
nodes["r"]=Zdiag
nodes
nodes.to_csv('diigo/nodes_diigo.csv')
edges.to_csv('diigo/edges_diigo.csv')<jupyter_output><empty_output><jupyter_text>Import Nodes and EdgesIf starting from existing saved files<jupyter_code>import pandas as pd
nodes = pd.read_csv('diigo/nodes_diigo.csv',index_col="Unnamed: 0")
nodes
# Filter out the tags that are only used once
nodes[nodes['r'] == 1]
nodes2 = nodes[nodes['r'] > 1]
nodes2
edges = pd.read_csv('Diigo/edges_diigo.csv',index_col="Unnamed: 0")
edges
edges[edges.source == 17]
nodes.iloc[18]<jupyter_output><empty_output><jupyter_text>Top 10 graph<jupyter_code>nodes = pd.read_csv('diigo/nodes_diigo_top10.csv')
edges = pd.read_csv('diigo/edges_diigo_top10.csv')<jupyter_output><empty_output><jupyter_text>Graph building<jupyter_code>import networkx as nx
G = nx.Graph()
for index, row in nodes.iterrows():
    # print(row['tag'],row['r'])
    # G.add_node(row['id'])
    G.add_node(index)
G.number_of_nodes()
G.nodes()
type(edges['source'][0])
# By default, source and target are numpy.int64
# Translated to int for further JSON serialization
for index, row in edges.iterrows():
    # print(row['source'],row['target'])
    G.add_edge(int(row['source']),int(row['target']))
# Check that we have not added nodes in the process of adding edges
G.number_of_nodes()<jupyter_output><empty_output><jupyter_text>D3 Force-directed graphhttps://observablehq.com/@d3/force-directed-graph<jupyter_code>import json
import flask
# import networkx as nx
from networkx.readwrite import json_graph
# nodes.loc[[0]].tag
nodes.tag[0]
# this d3 example uses the name attribute for the mouse-hover value,
# so add a name to each node
for n in G:
    # nodes id start at 0
    G.nodes[n]["name"] = nodes.tag[n]
    # G.nodes[n]["name"] = nodes.tag[n-1]
    # G.nodes[n]["name"] = n
    # print(nodes.tag[n])
    # print(n)
G.nodes[0]["name"]
# G.nodes.data()
# write json formatted data
d = json_graph.node_link_data(G)  # node-link format to serialize
# d
# d.get('nodes')
# d['nodes']
# type(d['nodes'][0]['id'])
type(d['links'][0]['target'])<jupyter_output><empty_output><jupyter_text>Pay attention when building JSON:*TypeError: Object of type int64 is not JSON serializable*<jupyter_code># write json
json.dump(d, open("force/force.json", "w"))
print("Wrote node-link JSON data to force/force.json")
# Serve the file over http to allow for cross origin requests
app = flask.Flask(__name__, static_folder="force")


@app.route("/")
def static_proxy():
    return app.send_static_file("force.html")


print("\nGo to http://localhost:8000 to see the example\n")
app.run(port=8000)<jupyter_output>Go to http://localhost:8000 to see the example

 * Serving Flask app '__main__' (lazy loading)
 * Environment: production
   WARNING: This is a development server. Do not use it in a production deployment.
   Use a production WSGI server instead.
 * Debug mode: off<jupyter_text>Simple representationRequires PIL and matplotlib<jupyter_code>nx.draw(G, with_labels=True, font_weight='bold')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14867773677736779, 'fraction_numerical': 0.017681426814268143, 'mean_word_length': 5.726990692864529, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 61, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '432799', 'license': 'MIT', 'n_tokens_mistral': 2434, 'n_tokens_neox': 2385, 'n_words': 585, 'path': 'knowledge.ipynb', 'repo_name': 'slevin48/knowledge', 'size': '6504'}