<jupyter_start><jupyter_text>Network Analysis Data RetrievalPull all the necessary data for building a graph of the social connections of the CaringBridge network. CSV data is stored in /home/shared/caringbridge/data/projects/sna-social-support/csv_data.<jupyter_code>import json
import numpy as np
import pandas as pd
from datetime import datetime
from IPython.display import display, clear_output
import csv as csv
import os
import subprocess
guestbook_fname = "/home/srivbane/shared/caringbridge/data/raw/guestbook_scrubbed.json"
journal_fname = "/home/srivbane/shared/caringbridge/data/raw/journal.json"
out_path = "/home/srivbane/shared/caringbridge/data/projects/sna-social-support/csv_data/"
chunk_size = 100000
allow = True<jupyter_output><empty_output><jupyter_text>GuestbookThe guestbook JSON has 829 chunks as of 2/14/2019<jupyter_code>if allow:
    processed_count = 0; chunk_count = 0; gb = [];
    header = ["from_userId", "siteId", "to_userId", "connectionType", "createdAt"]
    with open(guestbook_fname, encoding="utf-8") as infile, \
    open(os.path.join(out_path, "pcts.csv"), encoding="utf-8") as supp, \
    open(os.path.join(out_path, "gb.csv"), "w", newline="", encoding="utf-8") as outfile:
        csv_out = csv.writer(outfile)
        csv_out.writerow(header)
        pcts_r = csv.reader(supp)
        for line in infile:
            gb_json = json.loads(line)
            """
            supp.seek(0)
            for line in pcts_r:
                if (line[1] == gb_json["siteId"]):
            """
            gb.append((int(gb_json["userId"]), int(gb_json["siteId"]), "guestbook", int(gb_json["createdAt"]["$date"])))
            processed_count += 1
            if processed_count == chunk_size:
                chunk_count += 1
                csv_out.writerows(gb)
                clear_output(); display("" + str(chunk_count) + " chunks of " + str(chunk_size) + " processed.");
                gb = []; processed_count = 0;
        csv_out.writerows(gb)
display("Finished!")<jupyter_output><empty_output><jupyter_text>Solving the Guestbook Problem<jupyter_code>users = pd.read_csv("/home/srivbane/shared/caringbridge/data/projects/sna-social-support/csv_data/pcts.csv")
site_user_map = {}
for siteId, group in users.groupby(by='siteId', sort=False):
    userIds = tuple(group.userId.tolist())
    site_user_map[siteId] = userIds
processed = 0
with open(os.path.join(out_path, 'gb.csv'), 'r') as infile:
    with open(os.path.join(out_path, 'gb1.csv'), 'w') as outfile:
        csv_r = csv.reader(infile)
        csv_o = csv.writer(outfile)
        for line in csv_r:
            try:
                from_userId, siteId, c_type, createdAt = line
                for to_userId in site_user_map[int(siteId)]:
                    csv_o.writerow((from_userId, siteId, to_userId, c_type, createdAt)) # format the lines
                    processed += 1 
            except:
                pass<jupyter_output><empty_output><jupyter_text>JournalsThe journal JSON has 153 chunks as of 2/14/2019<jupyter_code>header = ["userId", "siteId", "connectionType", "createdAt"]
with open(journal_fname, encoding="utf-8") as infile:
    processed_count = 0;
    chunk_count = 0;
    jo = [];
    for line in infile:
        jo_json = json.loads(line)
        try:
            jo.append((int(jo_json["userId"]), int(jo_json["siteId"]), "journal", int(jo_json["createdAt"]["$date"])))
        except KeyError:
            pass
        finally:
            processed_count += 1
            if processed_count == chunk_size:
                chunk_count += 1
                clear_output(); display("" + str(chunk_count) + " chunks of " + str(chunk_size) + " processed.");           
                processed_count = 0;
with open(os.path.join(out_path, "jo.csv"), "w", encoding="utf-8") as outfile:
    j = pd.DataFrame(jo, columns = header);
    j = j.sort_values(by=["userId"]);
    j.to_csv(outfile, header=True, index=False)
display("Finished!")<jupyter_output><empty_output><jupyter_text>Journal RepliesThe journal JSON has 153 chunks as of 2/14/2019<jupyter_code>if allow:
    processed_count = 0; chunk_count = 0; jr = []; offset = 0;
    header = ["from_userId", "siteId", "to_userID", "connectionType", "createdAt"]
    with open(journal_fname, encoding="utf-8") as infile, open(os.path.join(out_path, "jr.csv"), "w", newline="") as outfile:
        csv_out = csv.writer(outfile)
        csv_out.writerow(header)
        for line in infile:
            jo_json = json.loads(line)
            try:
                jr_json = jo_json["replies"]
                for reply in jr_json:
                    reply_ts = int(1000 * datetime.strptime(reply["createdAt"], "%Y-%m-%d %H:%M:%S").timestamp())
                    journ_ts = jo_json["createdAt"]["$date"]
                    if reply_ts > journ_ts and offset == 0:
                        jr.append((int(reply["userId"]), int(jo_json["siteId"]), int(jo_json["userId"]), "reply", reply_ts))
                    else:
                        if (offset == 0):
                            offset = journ_ts - reply_ts
                        jr.append((int(reply["userId"]), int(jo_json["siteId"]), int(jo_json["userId"]), "reply", reply_ts + offset))
                        display("Timetraveling reply!")
                offset = 0;
            except KeyError:
                pass
            finally:
                processed_count += 1
                if processed_count == chunk_size:
                    chunk_count += 1
                    csv_out.writerows(jr)
                    clear_output(); display("" + str(chunk_count) + " chunks of " + str(chunk_size) + " processed.");
                    jr = []; processed_count = 0;
        csv_out.writerows(jr)
display("Finished!")<jupyter_output><empty_output><jupyter_text>AmpsThe journal JSON has 153 chunks as of 2/14/2019<jupyter_code>if allow:
    processed_count = 0; chunk_count = 0; amps = [];
    header = ["from_userId", "siteId", "to_userId", "connectionType", "createdAt"]
    with open(journal_fname, encoding="utf-8") as infile, open(os.path.join(out_path, "amps.csv"), "w", newline="") as outfile:
        csv_out = csv.writer(outfile)
        csv_out.writerow(header)
        for line in infile:
            jo_json = json.loads(line)
            try:
                if len(jo_json["amps"]) > 0:
                    for amp in jo_json["amps"]:
                        amps.append((amp, int(jo_json["siteId"]), int(jo_json["userId"]), "amps", int(jo_json["createdAt"]["$date"])))
            except KeyError:
                pass
            finally:
                processed_count += 1
                if processed_count == chunk_size:
                    chunk_count += 1
                    csv_out.writerows(amps)
                    clear_output(); display("" + str(chunk_count) + " chunks of " + str(chunk_size) + " processed.");           
                    amps = []; processed_count = 0;
        csv_out.writerows(amps)
display("Finished!")<jupyter_output><empty_output><jupyter_text>Mapping User ID to Site IDThe journal JSON has 153 chunks as of 2/14/2019.<jupyter_code>if allow:
    processed_count = 0; chunk_count = 0; lib = {}; pcts = []; count = 0
    with open(journal_fname, encoding="utf-8") as infile, \
    open(os.path.join(out_path, "a.csv"), "w", newline="") as outfile:
        csv_a = csv.writer(outfile)
        for line in infile:
            jo_json = json.loads(line)
            csv_a.writerow((jo_json["userId"], jo_json["siteId"]))
            processed_count += 1
            if processed_count == chunk_size:
                chunk_count += 1
                processed_count = 0
                clear_output(); display(str(chunk_count) + " / 153");
display("[ OK ] Decoding phase complete. Entering sorting phase.");
if allow:
    with open(os.path.join(out_path, "a.csv"), "r", newline="") as infile, \
    open(os.path.join(out_path, "srt.csv"), "w", newline="") as outfile:
        cmd = ["sort", "-k1", "-n", "-t,", os.path.join(out_path, "a.csv")]
        subprocess.call(cmd, stdout=outfile)
        clear_output(); display("[ OK ] Sorting phase complete. Entering reduction phase.");
    with open(os.path.join(out_path, "srt.csv"), "r", newline="") as infile, \
    open(os.path.join(out_path, "b.csv"), "w", newline="") as outfile:
        interim = csv.reader(infile)
        sort = csv.writer(outfile)
        uid = 0; sid = 0; tot = 0; processed_count = 0; chunk_count = 0;
        for row in interim:
            if (row[0] != uid or row[1] != sid):
                sort.writerow((uid, sid, tot))
                uid = row[0]; sid = row[1]
                tot = 0
            tot += 1
            processed_count += 1
            if processed_count == chunk_size:
                chunk_count += 1
                processed_count = 0
                clear_output(); display(str(chunk_count) + " / 153" );
display("[ OK ] Reduction phase complete. Let me tabulate the percentage for each tuple.");
c = 0
if allow:
    with open(os.path.join(out_path, "b.csv"), "r", newline="") as infile, \
    open(os.path.join(out_path, "pcts.csv"), "w", newline="") as outfile:
        interim = csv.reader(infile)
        fin = csv.writer(outfile)
        tot = 0; processed_count = 0; chunk_count = 0; rcd = {"uid" : 0, "sites" : [] ,"quants" : []}
        fin.writerow(("userId", "siteId", "numJournals", "pctJournals"))
        for row in interim:
            if (row[0] != rcd["uid"]):
                pct_flag = True
                for i in range(len(rcd["sites"])):
                    pct = int(rcd["quants"][i]) / tot
                    if (pct_flag and pct != 1.0):
                        c += 1
                        pct_flag = False
                    fin.writerow((rcd["uid"], rcd["sites"][i], int(rcd["quants"][i]), pct))
                rcd = {"uid" : row[0], "sites" : [], "quants" : []}
                tot = 0
            rcd["sites"].append(row[1]); rcd["quants"].append(row[2]); tot += int(row[2])
            processed_count += 1
            if processed_count == chunk_size:
                chunk_count += 1
                processed_count = 0
                clear_output(); display(str(chunk_count) + " chunks processed from reduced CSV." );
        pct_flag = True
        for i in range(len(rcd["sites"])):
            pct = int(rcd["quants"][i]) / tot
            if (pct_flag and pct != 1.0):
                c += 1
                pct_flag = False
            fin.writerow((rcd["uid"], rcd["sites"][i], int(rcd["quants"][i]), pct))
    os.remove(os.path.join(out_path, "a.csv")); os.remove(os.path.join(out_path, "b.csv")); os.remove(os.path.join(out_path, "srt.csv"));
display("[ OK ] Finished! " + str(100 * c / (chunk_size * chunk_count + len(rcd["sites"]))) + "% of tuples with non-exclusive authors.");<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1458966565349544, 'fraction_numerical': 0.013631758312609377, 'mean_word_length': 2.3564142194744977, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '210437', 'license': 'MIT', 'n_tokens_mistral': 3666, 'n_tokens_neox': 3465, 'n_words': 759, 'path': 'data_pulling/NetworkDataRetrieval.ipynb', 'repo_name': 'levon003/cscw-caringbridge-interaction-network', 'size': '10857'}