<jupyter_start><jupyter_text>Directories and whatnot<jupyter_code>DATA_DIR = os.getenv("DATA_DIR")
content_api = os.path.join(DATA_DIR, "content_api")
content_file = os.path.join(content_api,"content.json.gz")
labelled_file = os.path.join(content_api,"labelled.csv.gz")
count = 0
with gzip.open(content_file, "rt") as reader:
    content_generator = ijson.items(reader, prefix='item')
    for content_item in content_generator:
        count+=1
count<jupyter_output><empty_output><jupyter_text>Extract text, links etc from a `content.json.gz` export (from data science folder) Seems suspiciously small<jupyter_code>def extract_link_types(content_item):
    links = []
    related_links = []
    coll_links = []
    
    if content_item is not None:
        links = get_text(content_item['details'])
        related_links = []
        coll_links = []
        if 'ordered_related_items' in content_item['links'].keys():
            related_links = [related_item['base_path'] for related_item in
                             content_item['links']['ordered_related_items'] if
                             'base_path' in related_item.keys()]

        if 'documents' in content_item['links'].keys():
            coll_links = [document['base_path'] for document in content_item['links']['documents'] if
                          'base_path' in document.keys()]
        
    return links, related_links, coll_links
destination = os.path.join(content_api, "content_reduced.json.gz")
print("Start:",datetime.datetime.now().strftime("%H:%M:%S"))
with gzip.open(content_file, "rt") as reader, gzip.open(destination, 'wb') as writer:
    content_generator = ijson.items(reader, prefix='item')
    for i,content_item in content_generator:
        row = {}
        row['base_path'] = content_item['base_path']
        row['content_id'] = content_item['content_id']
        row['title'] = content_item['title']
        row['description'] = content_item['description']
        row['details'] = content_item['details']
        l1, l2, l3 = extract_link_types(content_item)
        row['embedded_links'] = l1
        row['related_links'] = l2
        row['coll_links'] = l3
        string_dict = json.dumps(row)
#         row_list.append(string_dict)
        writer.write("{}\n".format(string_dict).encode())
        if i % 10000 == 0:
            print("i:",i,datetime.datetime.now().strftime("%H:%M:%S"))
print("End:",datetime.datetime.now().strftime("%H:%M:%S"))
row_list = []
with gzip.open(destination, 'rt') as reader:
    for line in reader.readlines():
        row_list.append(line)
df1 = pd.DataFrame([json.loads(s) for s in row_list])
df1.shape
df1.head(2)
destination = os.path.join(content_api, "content_wlinks.csv.gz")
df_wlinks = df1[(df1.embdedded_links.str.len()>0) | (df1.related_links.str.len()>0) | (df1.coll_links.str.len()>0)]
df_wlinks.to_csv(destination, compression="gzip", index=False)
df1.to_csv(destination, compression="gzip", index=False)<jupyter_output><empty_output><jupyter_text>Set up `src_node` and `dest_node` pairs<jupyter_code>df_wlinks = df1[(df1.embdedded_links.str.len()>0) | (df1.related_links.str.len()>0) | (df1.coll_links.str.len()>0)]
df_wlinks.columns
def iterate_links(tupl, ltype_i, ltype_str):
    return [{'src_node': tupl.base_path,
               'dest_node': d,
               'link_type': ltype_str,
               'src_cid': tupl.content_id} for d in tupl[ltype_i]]
link_index_type = [(df_wlinks.columns.get_loc(lt), lt) for lt in ['coll_links', 
                                                                  'embdedded_links', 
                                                                  'related_links']]
link_index_type
rowlist = []
for tup in df_wlinks.itertuples(index=False):
    for lt, ltype in link_index_type:
        list_links = iterate_links(tup, lt, ltype)
        rowlist.extend(list_links)
df_struct_edges = pd.DataFrame(rowlist)
df_struct_edges.head()<jupyter_output><empty_output><jupyter_text>Figure out destination node `content_id`<jupyter_code>labelled = pd.read_csv(labelled_file, compression="gzip")
base_cid = dict(zip(labelled.base_path, labelled.content_id))
df_struct_edges['dest_cid'] = df_struct_edges['dest_node'].map(base_cid)
df_struct_edges.shape
### Check overlap of structural edges with labelled
diff_label_struct = set(labelled.content_id).difference(set(df_struct_edges.src_cid.values))
len(diff_label_struct)
df_struct_edges[df_struct_edges['dest_cid'].isna()].link_type.value_counts()<jupyter_output><empty_output><jupyter_text>Data seems insufficient, might as well extract base_paths in labelled to get all data from the content API<jupyter_code>with gzip.open(os.path.join(DATA_DIR, "processed_network", "labelled_for_conext.csv.gz"), "wt") as w:
    w.write("Node\n")
    for tup in labelled.itertuples():
        if tup.content_id in diff_label_struct:
            w.write("{}\n".format(tup.base_path))
temp = pd.read_csv(os.path.join(DATA_DIR, "processed_network", "labelled_for_conext.csv.gz"), compression="gzip")
temp.head()
temp.shape[0]/2
temp[0:87000].to_csv(os.path.join(DATA_DIR, "processed_network", "labelled_for_conext_pt1.csv.gz"), 
                      compression="gzip", index=False)
temp[87000:].to_csv(os.path.join(DATA_DIR, "processed_network", "labelled_for_conext_pt2.csv.gz"), 
                      compression="gzip", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1398549376976009, 'fraction_numerical': 0.008926910916868142, 'mean_word_length': 3.902461257976299, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '405944', 'license': 'MIT', 'n_tokens_mistral': 1833, 'n_tokens_neox': 1785, 'n_words': 296, 'path': 'notebooks/data_extract/extract_content_links_for_structnet_training_data.ipynb', 'repo_name': 'alphagov-mirror/govuk-network-embedding', 'size': '5377'}