<jupyter_start><jupyter_text>Create HathiTrust Zephir and US Federal Document Overlap Datset  Data**Zephir Notebook****This notebook requires the zephir-oclc-feddocs-dataset to run. This dataset has not been included in the repository for size reasons. There are two options if you do not have the dataset**  * Contact the Zephir team for the latest 'zephir-oclc-feddocs-dataset' dataset.  * Skip this notebook and use the included 'zephir-htusfd-overlap' dataset created by this notebook in futher analysis.  Imports<jupyter_code>import datetime
import collections
import gzip
import json
import os
import shutil


import pandas<jupyter_output><empty_output><jupyter_text>DatasetLoading  US Fed Docs Registry (OCLC numbers)The US Fed Docs Registry: https://github.com/HTGovdocs/feddoc_oclc_nums<jupyter_code>with open('data/feddoc_oclc_nums.manifest.json') as json_file:
    usfd_manifest = json.load(json_file)
    
print("US Fed Doc Registry data was commited at {}".format(usfd_manifest["datetime"]))

usfeddoc_registry_df = pandas.read_csv("data/feddoc_oclc_nums.txt",
    names=["oclc"],
    dtype=usfd_manifest["schema"],
    header = 0)<jupyter_output>US Fed Doc Registry data was commited at 2019-09-03T14:47:29<jupyter_text>Zephir Dataset (Repository-ingested only)This file is an export of Zephir data from the htmm database, and created by [jupyter notebook](http://localhost:8888/lab/tree/create_zephir_dataset.ipynb). The original file is not included in version control for size reasons.<jupyter_code>if not os.path.exists("data/zephir-oclc-feddocs-dataset.csv"):
    print("Check for dataset on Google Drive. It was too large for Github.")
    raise Exception("Missing dataset! See output for details.")
    
with open('data/zephir-oclc-feddocs-dataset.manifest.json') as json_file:
    zephir_manifest = json.load(json_file)
    
print("Zephir data was created {}".format(zephir_manifest["datetime"]))

zephir_df = pandas.read_csv("data/zephir-oclc-feddocs-dataset.csv", dtype=zephir_manifest["schema"])<jupyter_output>Zephir data was created 2019-09-03T19:00:53.931199<jupyter_text>Create overlap dataset  How many volumes in HathiTrust have overlapping OCLCs in the US Fed Docs Registry?<jupyter_code># get overlap
rows_with_matching_oclc = zephir_df["oclc"].isin(usfeddoc_registry_df["oclc"])
usfd_overlap_df = (zephir_df[rows_with_matching_oclc])

# only count unique volumes, some may have multiple oclcs and matches
volumes_matched = usfd_overlap_df.htid.unique()
print("There are {} volumes w/ OCLC numbers matched in the registry".format(len(volumes_matched)))<jupyter_output>There are 1227577 volumes w/ OCLC numbers matched in the registry<jupyter_text>Write dataset to overlap, no filtering or dedupingThis data is raw overlap. It includes matches (including duplicate voluemes) that may or may not be indicated as US Federal Documents in their MARC records (usfd_status).<jupyter_code>export_df = usfd_overlap_df.drop(columns="zdb_row").sort_values(by=["contributor_code", "contribsys_id"])
# save the dataset
dataset_name = "zephir-htusfd-overlap"
dataset_file ="data/{}.csv".format(dataset_name)
export_df.to_csv(dataset_file, header=True, index=False)

# gzip large file
with open(dataset_file, "rb") as f_in:
    with gzip.open("{}.gz".format(dataset_file), 'wb') as f_out:
        shutil.copyfileobj(f_in, f_out)<jupyter_output><empty_output><jupyter_text>Create manifest<jupyter_code>manifest = {}
manifest["name"] = dataset_name
manifest["description"] = "Dataset representing the OCLC overlap between Zephir OCLCS and the HT US Fed Docs OCLCS"
manifest["datetime"] = str(datetime.datetime.utcfromtimestamp(os.path.getmtime("data/{}.csv".format(manifest["name"]))).isoformat())

# derive schema from dataframe
schema = collections.OrderedDict()
for column in export_df.columns.array:
    schema[str(column)]= str(export_df.dtypes[column])
manifest["schema"] = schema

# format
manifest["format"] = {
        "delimiter": ",",
        "encoding": "utf8",
        "extension": "csv",
        "header": True,
        "type": "delimited"
    }
        
# note the origins
manifest["data-origins"] = [
    {
    "origin": "zephir-oclc-feddocs-dataset.csv",
    "datetime": str(zephir_manifest["datetime"])},
    {"origin": "feddoc_oclc_nums.txt",
    "datetime": str(usfd_manifest["datetime"])
    }
]

# save the manifest
manifest_file = "data/{}.manifest.json".format(dataset_name)
with open(manifest_file, 'w') as outfile:
    json.dump(manifest, outfile, indent=4)<jupyter_output><empty_output><jupyter_text>Finishing up!<jupyter_code>print("Completed notebook ({}).".format(datetime.datetime.utcnow().isoformat()))
print("Output created:")
print(dataset_file)
print(manifest_file)<jupyter_output>Completed notebook (2019-09-03T19:01:49.920570).
Output created:
data/zephir-htusfd-overlap.csv
data/zephir-htusfd-overlap.manifest.json
{'fraction_non_alphanumeric': 0.1162695152013147, 'fraction_numerical': 0.013968775677896467, 'mean_word_length': 5.926031294452347, 'pattern_counts': {'":': 9, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '397067', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1684, 'n_tokens_neox': 1590, 'n_words': 387, 'path': 'create_overlap_dataset.ipynb', 'repo_name': 'cdlib/zephir-htusfd-suggestions', 'size': '4868'}