<jupyter_start><jupyter_text>Data exploration<jupyter_code>to_df(graph.run("MATCH (n) RETURN distinct labels(n), count(*)"))
to_df(graph.run("MATCH (p:Publication) RETURN p.id AS id, p.title AS title"))<jupyter_output><empty_output><jupyter_text>Query relationships<jupyter_code>pub = 'http://arxiv.org/abs/1906.02739v1'
query = """
MATCH (p1:Publication)-[r]-(n)
WHERE p1.id = {pub}
RETURN p1.title, type(r), n.name
"""

to_df(graph.run(query, pub=pub))
query = """
MATCH (p1:Publication)-[]-(rn)-[]-(p2:Publication)
WHERE p1.id = {pub}
RETURN p1.id, p1.title, count(rn) as n_relations, p2.id, p2.title
ORDER BY n_relations DESC
LIMIT 10
"""

to_df(graph.run(query, pub=pub))<jupyter_output><empty_output><jupyter_text>Compute similarities<jupyter_code>query = """
MATCH (pub:Publication)-[]-(rel)
WITH {item: id(pub), categories: collect(id(rel))} as pubData
WITH collect(pubData) as data

CALL algo.similarity.jaccard(data, {concurrency:1, similarityCutoff: 0.1, write:false})
YIELD nodes, similarityPairs, write, writeRelationshipType, writeProperty, min, max, mean, stdDev, p25, p50, p75, p90, p95, p99, p999, p100

RETURN nodes, similarityPairs, write, writeRelationshipType, writeProperty, min, max, mean, p95
"""

to_df(graph.run(query, pub=pub))

query = """
MATCH (pub:Publication)-[]-(rel), (from:Publication {id: {pub}})
WITH {item: id(pub), categories: collect(id(rel))} as pubData, [id(from)] as sourceIds
WITH collect(pubData) as data, sourceIds

CALL algo.similarity.overlap.stream(data, {sourceIds: sourceIds, concurrency:1, similarityCutoff: 0.1})
YIELD item1, item2, similarity
WITH algo.getNodeById(item1) AS from, algo.getNodeById(item2) AS to, similarity

RETURN from.id, from.title, to.id, to.title, similarity
ORDER BY similarity DESC
LIMIT 100
"""

to_df(graph.run(query, pub=pub))

compared_to = [
    'http://arxiv.org/abs/1906.01792v1',
    'http://arxiv.org/abs/1906.02698v1',
    'http://arxiv.org/abs/1812.01601v3',
    'http://arxiv.org/abs/1906.02739v1',
]

query = """
MATCH (pub:Publication)-[]-(rel),
      (from:Publication),
      (to:Publication)
WHERE from.id = {pub} AND to.id IN {to}

WITH {item: id(pub), categories: collect(id(rel))} as pubData, [id(from)] as sourceIds, [id(to)] as targetIds
WITH collect(pubData) as data, sourceIds, targetIds

CALL algo.similarity.overlap.stream(data, {sourceIds: sourceIds, targetIds: targetIds, concurrency:1, similarityCutoff: 0.1})
YIELD item1, item2, similarity
WITH algo.getNodeById(item1) AS from, algo.getNodeById(item2) AS to, similarity

RETURN from.id, from.title, to.id, to.title, similarity
ORDER BY similarity DESC
LIMIT 100
"""

to_df(graph.run(query, pub=pub, to=compared_to))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1624722427831236, 'fraction_numerical': 0.038860103626943004, 'mean_word_length': 6.1697612732095495, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '55967', 'license': 'MIT', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1014, 'n_words': 228, 'path': 'notebooks/Neo4j explorer.ipynb', 'repo_name': 'Penguinazor/mse.wem.project', 'size': '2702'}