<jupyter_start><jupyter_text>Run this script to make sure that the make_clean_data functions are working properly. Warning some of them may take a while (particularly the functions that go through the whole edgelist).<jupyter_code>import sys

sys.path.append('../../code/')
import os
import json
from datetime import datetime
import time

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import scipy.stats as stats

import networkx as nx

%load_ext autoreload
%autoreload 2
%matplotlib inline

data_dir = '../../data/'
from make_clean_data import *
make_clean_case_metadata(data_dir)
start = time.time()
make_clean_edgelist(data_dir, overwrite=True)
end = time.time()
print 'took %d seconds' % (end - start)
make_clean_jurisdiction_file(data_dir)
make_jurisdiction_edgelist(data_dir)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08133971291866028, 'fraction_numerical': 0.0011961722488038277, 'mean_word_length': 5.860655737704918, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '190723', 'license': 'MIT', 'n_tokens_mistral': 277, 'n_tokens_neox': 265, 'n_words': 88, 'path': 'explore/Iain/test_pipeline.ipynb', 'repo_name': 'wethepeopleonline/law-net', 'size': '836'}