<jupyter_start><jupyter_text>Inspect and processing data manually<jupyter_code>%%sh
aws s3 cp s3://amazon-reviews-pds/tsv/amazon_reviews_us_Camera_v1_00.tsv.gz /tmp
import pandas as pd
import numpy as np
data = pd.read_csv('/tmp/amazon_reviews_us_Camera_v1_00.tsv.gz', sep='\t', 
                   compression='gzip', error_bad_lines=False, dtype='str')
data.dropna(inplace=True)
print(data.shape)
print(data.columns)
data.head()
data = data[:100000]
data = data[['star_rating', 'review_body']]
data.star_rating.unique()
data['label'] = data.star_rating.map({
    '1': '__label__negative__',
    '2': '__label__negative__',
    '3': '__label__neutral__',
    '4': '__label__positive__',
    '5': '__label__positive__'})
data = data.drop(['star_rating'], axis=1)
data = data[['label', 'review_body']]
data.head()
import nltk
nltk.download('punkt')
%%time
data['review_body'] = data['review_body'].apply(nltk.word_tokenize)
%%time
data['review_body'] = data.apply(lambda row: " ".join(row['review_body']).lower(), axis=1)
data.head()
from sklearn.model_selection import train_test_split

training, validation = train_test_split(data, test_size=0.05)
print(training.shape)
print(validation.shape)
np.savetxt('/tmp/training.txt', training.values, fmt='%s')
np.savetxt('/tmp/validation.txt', validation.values, fmt='%s')
!head -5 /tmp/training.txt<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>import sagemaker

print(sagemaker.__version__)

session = sagemaker.Session()
bucket = session.default_bucket()
# Run this cell if you want to use the data you processed manually

prefix = 'amazon-reviews'

s3_train_path = session.upload_data(path='/tmp/training.txt', bucket=bucket, key_prefix=prefix+'/input/train')
s3_val_path = session.upload_data(path='/tmp/validation.txt', bucket=bucket, key_prefix=prefix+'/input/validation')
s3_output = 's3://{}/{}/output/'.format(bucket, prefix)

print(s3_train_path)
print(s3_val_path)
print(s3_output)
# Run this cell if you want to use the data processed by SageMaker Processing

prefix = 'amazon-reviews'

s3_train_path = 
s3_val_path = 
s3_output = 's3://{}/{}/output/'.format(bucket, prefix)

print(s3_train_path)
print(s3_val_path)
print(s3_output)
from sagemaker.image_uris import retrieve

region = session.boto_session.region_name    
container = retrieve('blazingtext', region)
print(container)
bt = sagemaker.estimator.Estimator(
    container,
    sagemaker.get_execution_role(), 
    instance_count=1, 
    instance_type='ml.c5.2xlarge',
    output_path=s3_output)
bt.set_hyperparameters(mode='supervised')
train_data = sagemaker.TrainingInput(
    s3_train_path,
    content_type='text/plain')

validation_data = sagemaker.TrainingInput(
    s3_val_path,
    content_type='text/plain')

s3_channels = {'train': train_data, 'validation': validation_data}
bt.fit(inputs=s3_channels)
bt_predictor = bt.deploy(initial_instance_count=1, instance_type='ml.t2.medium')
import json
import pprint

sentences = ['This is a bad camera it doesnt work at all , i want a refund  .', 
             'The camera works , the pictures are decent quality, nothing special to say about it .',
             'Very happy to have bought this , exactly what I needed']

payload = {"instances" : sentences, "configuration": {"k": 3}}

bt_predictor.serializer = sagemaker.serializers.JSONSerializer()
response = bt_predictor.predict(payload)

print(response)
bt_predictor.delete_endpoint()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13876843018213356, 'fraction_numerical': 0.014455044810638912, 'mean_word_length': 5.865079365079365, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '95875', 'license': 'MIT', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1191, 'n_words': 234, 'path': 'Chapter 06/blazingtext/BlazingText on Amazon Reviews - Classification.ipynb', 'repo_name': 'amikewatson/Learn-Amazon-SageMaker-second-edition', 'size': '3459'}