<jupyter_start><jupyter_text>Amazon Comprehend<jupyter_code>comprehend = boto3.client(service_name='comprehend', region_name='us-east-1')
# Dominant Language
def get_language(text):
    if text != "":
        language_resp = comprehend.detect_dominant_language(Text = text)
        languages = language_resp['Languages']
        language_count = len(languages)
        return languages, language_count
    else:
        return None
# Named Entities
def get_named_entities(text):
    if text != "":
        entity_resp = comprehend.detect_entities(Text=text, LanguageCode='en')
        entities = entity_resp['Entities']
        entity_count = len(entities)
        return entities, entity_count
    else:
        return None
# Key Phrases
def get_key_phrases(text):
    if text != "":
        kp_resp = comprehend.detect_key_phrases(Text=text, LanguageCode='en')
        key_phrases = kp_resp['KeyPhrases']
        key_phrase_count = len(key_phrases)
        return key_phrases, key_phrase_count
    else:
        return None
# Sentiment
def get_sentiment(text):
    if text != "":
        sentiment_resp = comprehend.detect_sentiment(Text=text, LanguageCode='en')
        sentiment = sentiment_resp['Sentiment']
        sentiment_score_mixed = sentiment_resp['SentimentScore']['Mixed']
        sentiment_score_negative = sentiment_resp['SentimentScore']['Negative']
        sentiment_score_neutral = sentiment_resp['SentimentScore']['Neutral']
        sentiment_score_positive = sentiment_resp['SentimentScore']['Positive']
        return sentiment, sentiment_score_mixed, sentiment_score_negative, sentiment_score_neutral, sentiment_score_positive
    else:
        return None
# Get the language
md2['Language'] = md2['Description'].map(get_language)
md2.to_csv("yts_data_with_language.csv", encoding='utf-8')
# Get the named entities
md2['Entities'] = md2['Description'].map(get_named_entities)
md2.to_csv("yts_data_with_entities.csv", encoding='utf-8')
# Get the key phrases
md2['Key Phrases'] = md2['Description'].map(get_key_phrases)
md2.to_csv("yts_data_with_key_phrases.csv", encoding='utf-8')
# Get the sentiment
md2['Sentiment'] = md2['Description'].map(get_sentiment)
md2.to_csv("yts_data_fully_enriched.csv", encoding='utf-8')
record = json.loads(md2.head(1).to_json())
print(json.dumps(record, indent=2))
# Parse the language data and append to the dataframe
lang_codes = list()
lang_scores = list()

language_data = md2['Language'].tolist()
for item in language_data:
    try:
        lang_codes.append(item[0][0]['LanguageCode'])
        lang_scores.append(item[0][0]['Score'])
    except:
        lang_codes.append(None)
        lang_scores.append(None)
    
md2['Language Code'] = lang_codes
md2['Language Score'] = lang_scores
# Parse the sentiment data

sentiments = list()
mixed_scores = list()
negative_scores = list()
neutral_scores = list()
positive_scores = list()

sentiment_data = md2['Sentiment'].tolist()
for item in sentiment_data:
    try:
        sentiments.append(item[0])
        mixed_scores.append(item[1])
        negative_scores.append(item[2])
        neutral_scores.append(item[3])
        positive_scores.append(item[4])
    except:
        sentiments.append(None)
        mixed_scores.append(None)
        negative_scores.append(None)
        neutral_scores.append(None)
        positive_scores.append(None)
        
md2['Sentiment 2'] = sentiments
md2['Sentiment Mixed Score'] = mixed_scores
md2['Sentiment Negative Score'] = negative_scores
md2['Sentiment Neutral Score'] = neutral_scores
md2['Sentiment Positive Score'] = positive_scores

del md2['Sentiment']
md2.rename(columns={'Sentiment 2': 'Sentiment'}, inplace=True)
record = json.loads(md2.head(1).to_json())
print(json.dumps(record, indent=2))<jupyter_output><empty_output><jupyter_text>Convert the columns to the correct data types<jupyter_code>md2.dtypes
# Clean up the Downloads column for float conversion
def remove_comma(s):
    try:
        s = s.replace(",", "")
    except:
        pass
    return s
# Convert to floats
cols_to_convert_to_float = ['YTS Rating', 'Likes', 'IMDb Rating', 'Best Rating', 'Rating Count', 'Downloads']

for col in cols_to_convert_to_float:
    md2[col] = md2[col].apply(remove_comma)
    md2[col] = md2[col].astype('float64', inplace=True)
md2.dtypes<jupyter_output><empty_output><jupyter_text>Save the Final Result<jupyter_code># Save the final data file
md2.to_csv("yts_data_fully_enriched_clean.csv", index=False, encoding="utf-8")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12255772646536411, 'fraction_numerical': 0.012433392539964476, 'mean_word_length': 4.680958385876418, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '196571', 'license': 'Apache-2.0', 'n_tokens_mistral': 1490, 'n_tokens_neox': 1458, 'n_words': 274, 'path': 'aws_es/1 - YTS.am Scraper.ipynb', 'repo_name': 'rdempsey/osc-demo', 'size': '4504'}