<jupyter_start><jupyter_text>ProcessingIn this notebook, we read a wiki text snippet from an S3 bucket, process it and save the file abck on S3 to be imported into a Trino database and visualized on SuperSet<jupyter_code>import os
import nltk
import warnings

from dotenv import load_dotenv, find_dotenv
import boto3
from datasets import load_dataset
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

warnings.filterwarnings("ignore")
load_dotenv(find_dotenv())
## Create a .env file on your local with the correct configs
s3_endpoint_url = os.getenv("S3_ENDPOINT")
s3_access_key = os.getenv("S3_ACCESS_KEY")
s3_secret_key = os.getenv("S3_SECRET_KEY")
s3_bucket = os.getenv("S3_BUCKET")
# Create an S3 client
s3 = boto3.client(
    service_name="s3",
    aws_access_key_id=s3_access_key,
    aws_secret_access_key=s3_secret_key,
    endpoint_url=s3_endpoint_url,
)<jupyter_output><empty_output><jupyter_text>Read from S3<jupyter_code>raw_file_path = '../data/raw/wiki.txt'
processed_file_path = '../data/processed/token.txt'
key = 'op1-pipelines/wiki.txt'
s3.download_file(Bucket=s3_bucket, Key=key, Filename=raw_file_path)
f = open(raw_file_path, "r")
text_corpus = f.read()
text_corpus<jupyter_output><empty_output><jupyter_text>Process Text<jupyter_code>nltk.download('stopwords')
nltk.download('punkt')
stop_words = set(stopwords.words('english'))
word_tokens = word_tokenize(text_corpus)
 
tokenized = [w for w in word_tokens if not w.lower() in stop_words]
tokenized[0:10]
with open(processed_file_path, 'w') as file:
    for item in tokenized:
        file.write('%s\n' % item)<jupyter_output><empty_output><jupyter_text>Upload to S3<jupyter_code>key = 'op1-pipelines/processed/token.txt'
s3.upload_file(Bucket=s3_bucket, Key=key, Filename=str(processed_file_path))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10618500273672687, 'fraction_numerical': 0.015873015873015872, 'mean_word_length': 7.088495575221239, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '580471', 'license': 'FTL', 'n_tokens_mistral': 697, 'n_tokens_neox': 683, 'n_words': 140, 'path': 'notebooks/processing.ipynb', 'repo_name': 'oindrillac/op1st-pipeline-demo', 'size': '1827'}