<jupyter_start><jupyter_text>IngestIn this notebook, we read a wiki text snippet and save the data to an S3 bucket<jupyter_code>import os
import warnings

from dotenv import load_dotenv, find_dotenv
import boto3
from datasets import load_dataset

warnings.filterwarnings("ignore")
load_dotenv(find_dotenv())
## Create a .env file on your local with the correct configs
s3_endpoint_url = os.getenv("S3_ENDPOINT")
s3_access_key = os.getenv("S3_ACCESS_KEY")
s3_secret_key = os.getenv("S3_SECRET_KEY")
s3_bucket = os.getenv("S3_BUCKET")
# Create an S3 client
s3 = boto3.client(
    service_name="s3",
    aws_access_key_id=s3_access_key,
    aws_secret_access_key=s3_secret_key,
    endpoint_url=s3_endpoint_url,
)
# ! jupyter labextension install @jupyter-widgets/jupyterlab-manager
dataset = load_dataset('wikitext', 'wikitext-2-v1')
dataset
text = dataset['train']['text'][0:50]
text = list(filter(None, text))
text<jupyter_output><empty_output><jupyter_text>Upload to S3<jupyter_code>file_path = '../data/raw/wiki.txt'
with open(file_path, 'w') as file:
    for item in text:
        file.write('%s\n' % item)
key = 'op1-pipelines/wiki.txt'
s3.upload_file(Bucket=s3_bucket, Key=key, Filename=str(file_path))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11721907841552143, 'fraction_numerical': 0.021018593371059015, 'mean_word_length': 6.503030303030303, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '502509', 'license': 'FTL', 'n_tokens_mistral': 479, 'n_tokens_neox': 475, 'n_words': 95, 'path': 'notebooks/ingest.ipynb', 'repo_name': 'oindrillac/op1st-pipeline-demo', 'size': '1237'}