<jupyter_start><jupyter_text>Create nodes for corpus 2020 - referral_site<jupyter_code>files = [
    'corpus_2020_referral_sites.json', 
    'corpus_2020_referral_sites_level_1.json',
    'corpus_2020_referral_sites_level_2.json',
    'corpus_2020_referral_sites_level_3.json',
]

corpus_2020_referral_sites_level_from_0_to_3_level_included = {}

for f in files:
    loaded_data = load_json(os.path.join(_DATA_PATH, f))
    print(f'For file "{f}" -> load {len(loaded_data)} records')
    corpus_2020_referral_sites_level_from_0_to_3_level_included.update(loaded_data)
len(corpus_2020_referral_sites_level_from_0_to_3_level_included)
list(corpus_2020_referral_sites_level_from_0_to_3_level_included.items())[:5]
corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES = []

for base_url, referral_sites in corpus_2020_referral_sites_level_from_0_to_3_level_included.items():
    if not referral_sites:
        corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES.append((base_url, base_url))

    for referral_site, _ in referral_sites:
        if referral_site != base_url:
            corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES.append((base_url, referral_site))
print(f'Node from referral_sites for corpus 2020: {len(corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES)}')
corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES[:10]<jupyter_output><empty_output><jupyter_text>Check how much sites need to be processed<jupyter_code>import redis 

red = redis.Redis()

referral_sites = set()

for l, r in corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES:
    referral_sites.add(l)
    referral_sites.add(r)

print('unique referral_sites:', len(referral_sites))

urls_that_needs_processing = [el for el in referral_sites if not red.get(el)]

print('Number of sites that need to be processed:', len(urls_that_needs_processing))<jupyter_output>Number of sites that need to be processed: 8586<jupyter_text>Load audience_ovelap sites<jupyter_code>corpus_2020_audience_overlap_from_0_to_3_levels_included = load_level_data(os.path.join(_DATA_PATH, 'corpus_2020_audience_overlap_sites_scrapping_result.json'), level=3)
len(corpus_2020_audience_overlap_from_0_to_3_levels_included)
corpus_2020_audience_overlap_from_0_to_3_levels_included['villagevoice.com']
corpus_2020_audience_overlap_from_0_to_3_levels_included_NODES = create_nodes(corpus_2020_audience_overlap_from_0_to_3_levels_included)
len(corpus_2020_audience_overlap_from_0_to_3_levels_included_NODES)
corpus_2020_audience_overlap_from_0_to_3_levels_included_NODES[:10]
type(corpus_2020_audience_overlap_from_0_to_3_levels_included_NODES)
type(corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES)
combined_nodes_from_0_to_3_level_included = corpus_2020_referral_sites_level_from_0_to_3_level_included_NODES + corpus_2020_audience_overlap_from_0_to_3_levels_included_NODES
len(combined_nodes_from_0_to_3_level_included)
unique_nodes = set()

for l, r in combined_nodes_from_0_to_3_level_included:
    unique_nodes.add(l)
    unique_nodes.add(r)
len(unique_nodes)
print('Sites that needs to extract alexa section info: ', len([el for el in unique_nodes if not red.get(el)]))<jupyter_output>Sites that needs to extract alexa section info:  46134<jupyter_text>---<jupyter_code>def combined_nodes_referral_sites_audience_overlap(year='2020', level=1):
    files = [
        'corpus_2020_referral_sites.json', 
        'corpus_2020_referral_sites_level_1.json',
        'corpus_2020_referral_sites_level_2.json',
        'corpus_2020_referral_sites_level_3.json',
    ]

    referral_sites = {}

    for f in files[:level+1]:
        loaded_data = load_json(os.path.join(_DATA_PATH, f))
        print(f'For file "{f}" -> load {len(loaded_data)} records')
        referral_sites.update(loaded_data)

    referral_sites_NODES = []

    for base_url, referral_sites in referral_sites.items():
        if not referral_sites:
            referral_sites_NODES.append((base_url, base_url))

        for referral_site, _ in referral_sites:
            if referral_site != base_url:
                referral_sites_NODES.append((base_url, referral_site))

    audience_overlap_sites = load_level_data(os.path.join(_DATA_PATH, 'corpus_2020_audience_overlap_sites_scrapping_result.json'), level=level)

    audience_overlap_sites_NODES = create_nodes(audience_overlap_sites)

    print('referral_sites node size:', len(referral_sites_NODES), 'audience_overlap node size:', len(audience_overlap_sites_NODES))

    return audience_overlap_sites_NODES + referral_sites_NODES
d = combined_nodes_referral_sites_audience_overlap(level=)
import redis

red = redis.Redis()
unique_nodes = set()

for l, r in d:
    unique_nodes.add(l)
    unique_nodes.add(r)

print('Sites that needs to extract alexa section info: ', len([el for el in unique_nodes if not red.get(el)]))
from utils.notebook_utils import get_alexa_information_sections
alexa_data_for_nodes_level_0_and_1 = get_alexa_information_sections(unique_nodes)
from datetime import datetime

now = datetime.now()

current_time = now.strftime("%H:%M:%S")
print("Current Time =", current_time)

result = list(zip(unique_nodes, alexa_data_for_nodes_level_0_and_1))
from datetime import datetime

now = datetime.now()

current_time = now.strftime("%H:%M:%S")
print("Current Time =", current_time) # started 5:02 - 05:27:49 for 2500
result[:5]
import json


with open('corpus_2020_levels_0_1_2_data.json', 'w') as f:
    json.dump(dict(result), f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0820882088208821, 'fraction_numerical': 0.04284428442844285, 'mean_word_length': 7.2925373134328355, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '194499', 'license': 'MIT', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2042, 'n_words': 290, 'path': 'notebooks/generate_node_features/Normalize node features for corpus 2020 for level 0 1 2 3.ipynb', 'repo_name': 'PacoPacov/site_similarity', 'size': '5555'}