<jupyter_start><jupyter_text>Compare to JGolbeck et al. on Fake News dataset<jupyter_code>golbeck_df = pd.read_csv("../data/jgolbeck.csv")
golbeck_df.shape<jupyter_output><empty_output><jupyter_text>Compare to Guess, Nyhan, and Reifler<jupyter_code>nyhan_df = pd.read_csv("../data/guess_nyhan_reifler.csv")
nyhan_df.shape<jupyter_output><empty_output><jupyter_text>Starbird Alternative Media<jupyter_code>starbird_df = pd.read_excel("../data/starbird_AlternativeDomains.xlsx")
starbird_df["domain"] = starbird_df["Domain"]

starbird_alt = starbird_df[starbird_df["Media Type (Determined through Content Analysis)"] == "Alternative Media"]
starbird_alt.shape<jupyter_output><empty_output><jupyter_text>Merged Fake News List<jupyter_code>sets = dict([
    ("golbeck", set(golbeck_df["domain"])),
    ("starbird_alt", set(starbird_alt["domain"])),
    ("nyhan", set(nyhan_df["domain"])),
])

# Dataset used for summer 2018
sets["merged"] = sets["nyhan"].union(sets["golbeck"]).union(sets["starbird_alt"])
for t, s in sets.items():
    print(t, len(s))
n = len(sets)
sim = np.zeros((n, n))

sets_list = list(sets.items())

for i in range(n):
    left, l = sets_list[i]
    for j in range(n):
        right, r = sets_list[j]
        print(left, right)
        o = len(l.intersection(r)) / len(l)
        print("\tOverlap:", o, len(l.intersection(r)), len(l))
        
        sim[i, j] = o
        
        
fig, ax = plt.subplots()
ax.imshow(sim)

ax.set_xticks(range(n))
ax.set_xticklabels([x[0] for x in sets_list], rotation=90)

ax.set_yticks(range(n))
ax.set_yticklabels([x[0] for x in sets_list])

plt.show()
sim
with open("summer2018_merged_tld_set.csv", "w") as out_file:
    for tld in sets["merged"]:
        out_file.write("%s\n" % tld)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16422121896162528, 'fraction_numerical': 0.006772009029345372, 'mean_word_length': 5.092783505154639, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '584774', 'license': 'MIT', 'n_tokens_mistral': 718, 'n_tokens_neox': 694, 'n_words': 120, 'path': 'notebooks/01_GraphOverlap.ipynb', 'repo_name': 'cbuntain/FakeNewsIConf2019', 'size': '1772'}