<jupyter_start><jupyter_text>Raw Corpora Intake - One OffWith some shortcuts since I'm mostly pulling from HermitDave's GitHub stash.  Imports and Settings<jupyter_code>from nate_givens_toolkit import cloud_io as cloud
# from nate_givens_toolkit import local_io as local
import pandas as pd
from datetime import datetime
#import requests
#import boto3<jupyter_output><empty_output><jupyter_text>Global Variables<jupyter_code>LOCAL_DIR = '/data/lexgen/raw_corpora/'
BUCKET_DIR = 'raw_corpora/'
BUCKET = 'lexgen'<jupyter_output><empty_output><jupyter_text>Logic  Specify Raw Corpora Variables<jupyter_code>url_filename = 'en_full.txt'
url_path = 'https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/en/'
filename = 'en_full_2018.txt'
lang_code = 'en'
source_url = 'https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/en/en_full.txt'
last_load_dtime = datetime.utcnow()
note = 'HermitDave\'s version of the full 2018 English file from OpenSubtitles'<jupyter_output><empty_output><jupyter_text>Read In Raw Corpora Table<jupyter_code>raw_corpora_inventory = cloud.read_csv_from_s3('raw_corpora_inventory.dat', 'data_files/', BUCKET, sep='|')
raw_corpora_inventory.head()<jupyter_output><empty_output><jupyter_text>Check raw_corpora_inventory for Pre-existing Duplicates<jupyter_code>can_proceed = True
if filename in raw_corpora_inventory['filename'].unique():
    can_proceed = False
    print(f'filename {filename} already exists in raw_corpora_inventory.')
if source_url in raw_corpora_inventory['source_url'].unique():
    can_proceed = False
    print(f'source_url {source_url} already exists in raw_corpora_inventory.')<jupyter_output><empty_output><jupyter_text>Copy File Locally<jupyter_code>if can_proceed:
    cloud.pull_data_from_url(url_filename, url_path, LOCAL_DIR, local_filename=filename, overwrite=True)<jupyter_output><empty_output><jupyter_text>Copy File to S3<jupyter_code>if can_proceed:
    cloud.push_file_to_s3(filename, LOCAL_DIR, BUCKET_DIR, BUCKET, overwrite=True)<jupyter_output><empty_output><jupyter_text>Update raw_corpora_inventory<jupyter_code>new_row = {'filename': filename, 'lang_code':lang_code, 'source_url':source_url, 'last_load_dtime':str(last_load_dtime), 'note':note}
raw_corpora_inventory = raw_corpora_inventory.append(new_row, ignore_index=True)
raw_corpora_inventory.head()<jupyter_output><empty_output><jupyter_text>Write Raw Corpora Back to S3<jupyter_code>cloud.write_csv_to_s3('raw_corpora_inventory.dat', BUCKET_DIR, BUCKET, raw_corpora_inventory, sep='|', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10299769408147579, 'fraction_numerical': 0.008455034588777863, 'mean_word_length': 9.756198347107437, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '214535', 'license': 'MIT', 'n_tokens_mistral': 983, 'n_tokens_neox': 941, 'n_words': 149, 'path': '.ipynb_checkpoints/Raw_Corpora_Intake_OneOff-checkpoint.ipynb', 'repo_name': 'nategivens/LexGen', 'size': '2602'}