<jupyter_start><jupyter_text>Inspect and processing data manually<jupyter_code>%%sh
aws s3 cp s3://amazon-reviews-pds/tsv/amazon_reviews_us_Camera_v1_00.tsv.gz /tmp
import pandas as pd
data = pd.read_csv('/tmp/amazon_reviews_us_Camera_v1_00.tsv.gz', sep='\t', compression='gzip',
                   error_bad_lines=False, dtype='str')
data.head()
data = data.dropna()
print(data.shape)
data = data[:100000]
data = data[['review_body']]
data.head()
import spacy

spacy_nlp = spacy.load('en')

def tokenize(text):
    tokens = spacy_nlp.tokenizer(text)
    tokens = [ t.text for t in tokens ]
    return " ".join(tokens).lower()
%%time
data['review_body'] = data['review_body'].apply(tokenize)
data.head()
import numpy as np

np.savetxt('/tmp/training.txt', data.values, fmt='%s')
!head -5 /tmp/training.txt<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>import boto3
import sagemaker

print(sagemaker.__version__)

session = sagemaker.Session()
bucket = session.default_bucket()
# Run this cell if you want to use the data you processed manually

prefix = 'amazon-reviews-word2vec'

s3_train_path = session.upload_data(path='/tmp/training.txt', bucket=bucket, key_prefix=prefix+'/input/train')
s3_output = 's3://{}/{}/output/'.format(bucket, prefix)

print(s3_train_path)
print(s3_output)
# Run this cell if you want to use the data processed by SageMaker Processing

prefix = 'amazon-reviews-word2vec'

s3_train_path = 
s3_output = 's3://{}/{}/output/'.format(bucket, prefix)

print(s3_train_path)
print(s3_output)
from sagemaker import image_uris

region = boto3.Session().region_name    
container = image_uris.retrieve('blazingtext', region)
print(container)
role = sagemaker.get_execution_role()

bt = sagemaker.estimator.Estimator(container,
                                   role, 
                                   instance_count=1, 
                                   instance_type='ml.p3.2xlarge',
                                   output_path=s3_output)
bt.set_hyperparameters(mode='skipgram', subwords=True)
train_data = sagemaker.TrainingInput(s3_train_path, 
                      distribution='FullyReplicated', 
                      content_type='text/plain',
                      s3_data_type='S3Prefix')

s3_channels = {'train': train_data}
bt.fit(inputs=s3_channels)
%%bash -s "$s3_output"
aws s3 ls --recursive $1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12746331236897274, 'fraction_numerical': 0.017190775681341718, 'mean_word_length': 3.66015625, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '235871', 'license': 'MIT', 'n_tokens_mistral': 842, 'n_tokens_neox': 805, 'n_words': 150, 'path': 'sdkv2/ch6/blazingtext/BlazingText on Amazon Reviews - Word2Vec.ipynb', 'repo_name': 'absynthe/Learn-Amazon-SageMaker', 'size': '2385'}