<jupyter_start><jupyter_text>Inspect and processing data manually<jupyter_code>%%sh
pip -q install gensim nltk
import pandas as pd
num_lines = 10000

data = pd.read_csv('abcnews-date-text.csv.gz', compression='gzip',
                      error_bad_lines=False, dtype='str', nrows=num_lines)

data = data.sample(frac=1)
data.head()
data = data.drop(['publish_date'], axis=1)
import string
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer 
    
stop_words = stopwords.words('english')
wnl = WordNetLemmatizer()

def process_text(text):
    for p in string.punctuation:
        text = text.replace(p, '')
    text = ''.join([c for c in text if not c.isdigit()])
    text = text.lower().split()
    text = [w for w in text if not w in stop_words] 
    text = [wnl.lemmatize(w) for w in text]
    return text
%%time
data['headline_text'] = data['headline_text'].apply(process_text)
data.head()
%%time

from gensim import corpora
dictionary = corpora.Dictionary(data['headline_text'])
print(dictionary)
dictionary.filter_extremes(keep_n=512)
print(dictionary)
with open('vocab.txt', 'w') as f:
    for index in range(0,len(dictionary)):
        f.write(dictionary.get(index)+'\n')
%%time

data['tokens'] = data.apply(lambda row: dictionary.doc2bow(row['headline_text']), axis=1)
data = data.drop(['headline_text'], axis=1)
data.head()
import io, boto3
import sagemaker
import sagemaker.amazon.common as smac
from scipy.sparse import lil_matrix

print(sagemaker.__version__)

session = sagemaker.Session()
bucket = session.default_bucket()
prefix = 'headlines-lda-ntm'
def build_protobuf_dataset(data, dictionary):
    num_lines = data.shape[0]
    num_columns = len(dictionary)
    token_matrix = lil_matrix((num_lines, num_columns)).astype('float32')
    line = 0
    for _, row in data.iterrows():
        for token_id, token_count in row['tokens']:
            token_matrix[line, token_id] = token_count
        line+=1
        
    buf = io.BytesIO()
    smac.write_spmatrix_to_sparse_tensor(buf, token_matrix, None)
    return buf
def upload_protbuf_dataset(buf, bucket, prefix, key):
    obj = '{}/{}'.format(prefix, key)
    buf.seek(0)
    boto3.resource('s3').Bucket(bucket).Object(obj).upload_fileobj(training_buf)
    path = 's3://{}/{}'.format(bucket,obj)
    return path
%%time
training_buf = build_protobuf_dataset(data, dictionary)
s3_training_path = upload_protbuf_dataset(training_buf, bucket, prefix, 'training/training.protobuf')
print(s3_training_path)
s3_auxiliary_path = session.upload_data(path='vocab.txt', key_prefix=prefix + '/input/auxiliary')
print(s3_auxiliary_path)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>s3_output = 's3://{}/{}/output/'.format(bucket, prefix)

print(s3_output)
import boto3
from sagemaker import image_uris

region = boto3.Session().region_name    
container = image_uris.retrieve('ntm', region)
print(container)
role = sagemaker.get_execution_role()

ntm = sagemaker.estimator.Estimator(container,
                                   role, 
                                   instance_count=1, 
                                   instance_type='ml.p3.2xlarge',
                                   sagemaker_session=session)
ntm.set_hyperparameters(num_topics=10, 
                        feature_dim=len(dictionary),
                        optimizer='adam', 
                        mini_batch_size=256, 
                        num_patience_epochs=10)
ntm.fit(inputs={'train': s3_training_path,
                'auxiliary': s3_auxiliary_path})
topics = ['crime','legal','disaster','sports','unknown1',
          'international','local','finance','politics', 
          'unknown2']
ntm_predictor = ntm.deploy(initial_instance_count=1, instance_type='ml.t2.large')
def process_samples(samples, dictionary):
    num_lines = len(samples)
    num_columns = len(dictionary)
    sample_matrix = lil_matrix((num_lines, num_columns)).astype('float32')
    for line in range(0, num_lines):
        s = samples[line]
        s = process_text(s)
        s = dictionary.doc2bow(s)
        for token_id, token_count in s:
            sample_matrix[line, token_id] = token_count
        line+=1
    buf = io.BytesIO()
    smac.write_spmatrix_to_sparse_tensor(buf, sample_matrix, None)
    buf.seek(0)
    return buf
# Run this cell to try your own samples

samples = [
    "Major tariffs expected to end Australian barley trade to China",
    "US woman wanted over fatal crash asks for release after coronavirus halts extradition",
    "Fifty trains out of service as fault forces Adelaide passengers to 'pack like sardines",
    "Germany's Bundesliga plans its return from lockdown as football world watches",
    "All AFL players to face COVID-19 testing before training resumes"
]
# Run this cell to load random samples from the dataset

data = pd.read_csv('abcnews-date-text.csv.gz', compression='gzip',
                      error_bad_lines=False, dtype='str', nrows=num_lines)
samples = data.sample(frac=0.001)
samples = np.array(samples.headline_text)
print(samples)
ntm_predictor.content_type = 'application/x-recordio-protobuf'

response = ntm_predictor.predict(process_samples(samples, dictionary))
print(response)
import json
import numpy as np

response = json.loads(response)

for r in response['predictions']:
    sorted_indexes = np.argsort(r['topic_weights']).tolist()
    sorted_indexes.reverse()
    top_topics = [topics[i] for i in sorted_indexes]
    top_weights = [r['topic_weights'][i] for i in sorted_indexes]
    pairs = list(zip(top_topics, top_weights))
    print(pairs[:3])
import pprint

pprint.pprint(response)
ntm_predictor.delete_endpoint()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11730769230769231, 'fraction_numerical': 0.010839160839160839, 'mean_word_length': 3.836010143702451, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '311058', 'license': 'MIT', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1835, 'n_words': 404, 'path': 'sdkv2/ch6/lda-ntm/NTM on ABC Headlines.ipynb', 'repo_name': 'absynthe/Learn-Amazon-SageMaker', 'size': '5720'}