<jupyter_start><jupyter_text>Export Articles (Optional) This notebook will export articles collected by the previous notebook to zip archives for storage and preprocessing.For more information and further instructions, consult the Chomp documentation at https://github.com/seangilleran/we1s_chomp.   INFO __authors__    = '<NAME>'  __copyright__  = 'copyright 2019, The WE1S Project'  __license__    = 'MIT'  __version__    = '0.1.0'  __email__      = '<EMAIL>'   SETTINGS<jupyter_code>import json
from pathlib import Path
from zipfile import ZipFile


project_dir = Path.home() / "write" / "dev" / "we1s_chomp"
article_dir = project_dir / "data" / "json" / "articles"
export_dir = project_dir / "data" / "export"

# Make the export directory if it does not already exist.
if not export_dir.exists():
    export_dir.mkdir(parents=True)
    
print(f"Loading articles from {article_dir}.")
print(f"Exporting archives to {export_dir}.\n\n")<jupyter_output><empty_output><jupyter_text>EXPORT<jupyter_code>archives = {}

for filename in article_dir.glob("**/*.json"):
    with open(filename, encoding="utf-8") as jsonfile:
        article = json.load(jsonfile)
        articles.append(article)
        
        # Store "no-exact-match" articles in separate archives.
        archive_name = article["query"]
        if "no-exact-match" in str(filename):
            archive_name += "(no-exact-match)"
        
        if not archives.get(archive_name, None):
            archives[archive_name] = []
        archives[archive_name].append(article)

for archive_name in archives:
    archive_filename = export_dir / f"{archive_name}.zip"
    print(f"Creating archive {archive_filename}...")
    with ZipFile(archive_filename, "w") as zipfile:
        for article in archives[archive_name]:
            original_filename = article_dir / f'{article["name"]}.json'
            new_filename = f'{article["name"]}.json'
            zipfile.write(original_filename, new_filename)
            print(f"- {new_filename}")
    print("Done!\n\n")
print("\n\n----------Time----------")<jupyter_output><empty_output><jupyter_text>SEE EXPORTED FILES<jupyter_code># TODO: link to dir<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1269986295111923, 'fraction_numerical': 0.005025125628140704, 'mean_word_length': 3.610526315789474, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '523623', 'license': 'MIT', 'n_tokens_mistral': 695, 'n_tokens_neox': 673, 'n_words': 160, 'path': 'notebooks/03_export.ipynb', 'repo_name': 'seangilleran/we1s_chomp', 'size': '2207'}