<jupyter_start><jupyter_text>Hotel Recommender SystemTrain and deploy a Hotel Recommender System using the public Expedia data from Kaggle competition https://www.kaggle.com/c/expedia-hotel-recommendations/overview**The goal is to help Expedia visitors find their dream hotel under cold start conditions!**It's a hard problem as we don't have historical information for new hotel clusters and we want to offer to customer more diversity when it comes to trip planning.Download data from https://www.kaggle.com/c/expedia-hotel-recommendations/data and unzip the downloaded file.<jupyter_code>import pandas as pd
import numpy as np
df = pd.read_csv('./expedia-hotel-recommendations/train.csv')
df.head(100)
df.dtypes
df['date_time'].head(10)
df['ts'] = pd.to_datetime(df['date_time'], format="%Y-%m-%d %H:%M:%S").values.astype(np.int64) // 10**6
df.dtypes
df.head(10)<jupyter_output><empty_output><jupyter_text>Let's split dataset into train/test<jupyter_code>from sklearn.model_selection import train_test_split

train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)<jupyter_output><empty_output><jupyter_text>Let's remove some hotel clusters to emulate the case of new items (i.e. hotel clusters)<jupyter_code>unique_items = df['hotel_cluster'].unique()
len(unique_items)<jupyter_output><empty_output><jupyter_text>Build the user-item interactions data set<jupyter_code>df_interactions = train_df[['user_id', 'hotel_cluster', 'ts']]
df_interactions.columns = ['USER_ID','ITEM_ID', 'TIMESTAMP']
df_interactions.head(10)
import boto3

import json
import numpy as np
import pandas as pd
import time

session = boto3.Session(profile_name='personalize')  # replace with an aws profile with access to S3 and Personalize
personalize = session.client('personalize', region_name='us-east-1')
personalize_runtime = session.client('personalize-runtime', region_name='us-east-1')
bucket = "personalize-hotels"  # replace with the name of your S3 bucket. Make sure the bucket is already created.
filename = "hotels-interactions.csv"  # replace with a name that you want to save the dataset under
# Save user-item interactions data set in a file locally
df_interactions.to_csv(filename, index=False)

# Upload user-item interactions data set file to S3
session.resource('s3').Bucket(bucket).Object(filename).upload_file(filename)<jupyter_output><empty_output><jupyter_text>Let's add metadata (i.e. features in terms of Machine Learning)In our case, we can add features for hotel clusters and/or features related to the context of a user<jupyter_code>df_item_metadata = train_df[['hotel_cluster', 'hotel_continent', 'hotel_country', 'hotel_market']]

df_item_metadata.columns = ['ITEM_ID', 'HOTEL_CONTINENT', 'HOTEL_COUNTRY', 'HOTEL_MARKET']
filename_item_metadata = "hotel-metadata"
df_item_metadata.to_csv(filename_item_metadata, index=False)

# Upload item metadata file to S3
session.resource('s3').Bucket(bucket).Object(filename_item_metadata).upload_file(filename_item_metadata)<jupyter_output><empty_output><jupyter_text>Time for user/context metadata/features<jupyter_code>df_user_metadata = train_df[
    [
        'user_id', 'user_location_country', 'user_location_region', 'user_location_city',
        'srch_adults_cnt', 'srch_children_cnt'
    ]
]

df_user_metadata.columns = [
    'USER_ID', 'USER_LOCATION', 'USER_LOCATION_REGION', 'USER_LOCATION_CITY',
    'SRCH_ADULTS_CNT', 'SRCH_CHILDREN_CNT'
]
filename_user_metadata = "user-metadata.csv"
df_user_metadata.to_csv(filename_user_metadata, index=False)

# Upload item metadata file to S3
session.resource('s3').Bucket(bucket).Object(filename_user_metadata).upload_file(filename_user_metadata)<jupyter_output><empty_output><jupyter_text>Define user-item interactions/item-metadata/user-metadata data set schema in Amazon Personalize<jupyter_code>schema = {
    "type": "record",
    "name": "Interactions",
    "namespace": "com.amazonaws.personalize.schema",
    "fields": [
        {
            "name": "USER_ID",
            "type": "string"
        },
        {
            "name": "ITEM_ID",
            "type": "string"
        },
        {
            "name": "TIMESTAMP",
            "type": "long"
        }
    ],
    "version": "1.0"
}

create_schema_response = personalize.create_schema(
    name = "hotel-recommender-cold-start-schema-v2",
    schema = json.dumps(schema)
)

schema_arn = create_schema_response['schemaArn']
print(json.dumps(create_schema_response, indent=2))
metadata_schema = {
    "type": "record",
    "name": "Items",
    "namespace": "com.amazonaws.personalize.schema",
    "fields": [
    {
        "name": "ITEM_ID",
        "type": "string"
    },
    {
        "name": "HOTEL_CONTINENT",
        "type": "int",
        "categorical": True
    },
    {
        "name": "HOTEL_COUNTRY",
        "type": "int",
        "categorical": True
    },
    {
        "name": "HOTEL_MARKET",
        "type": "int",
        "categorical": True
    }
    ],
    "version": "1.0"
}

create_metadata_schema_response = personalize.create_schema(
    name = "hotel-recommender-item-metadata-cold-start-schema-v2",
    schema = json.dumps(metadata_schema)
)

metadata_schema_arn = create_metadata_schema_response['schemaArn']
print(json.dumps(create_metadata_schema_response, indent=2))
user_metadata_schema = {
  "type": "record",
  "name": "Users",
  "namespace": "com.amazonaws.personalize.schema",
  "fields": [
      {
          "name": "USER_ID",
          "type": "string"
      },
      {
          "name": "USER_LOCATION",
          "type": "int",
          "categorical": True
      },
      {
          "name": "USER_LOCATION_REGION",
          "type": "int",
          "categorical": True
      },
      {
          "name": "USER_LOCATION_CITY",
          "type": "int",
          "categorical": True
      },
      {
          "name": "SRCH_ADULTS_CNT",
          "type": "int"
      },
      {
          "name": "SRCH_CHILDREN_CNT",
          "type": "int"
      }
  ],
  "version": "1.0"
}

create_user_metadata_schema_response = personalize.create_schema(
    name = "hotel-recommender-user-metadata-cold-start-schema-v2",
    schema = json.dumps(user_metadata_schema)
)

user_metadata_schema_arn = create_user_metadata_schema_response['schemaArn']
print(json.dumps(create_user_metadata_schema_response, indent=2))<jupyter_output>{
  "schemaArn": "arn:aws:personalize:us-east-1:296654805457:schema/hotel-recommender-user-metadata-cold-start-schema-v2",
  "ResponseMetadata": {
    "RequestId": "783323c5-0968-4f76-a98d-92384d654f76",
    "HTTPStatusCode": 200,
    "HTTPHeaders": {
      "content-type": "application/x-amz-json-1.1",
      "date": "Mon, 25 Nov 2019 05:37:05 GMT",
      "x-amzn-requestid": "783323c5-0968-4f76-a98d-92384d654f76",
      "content-length": "118",
      "connection": "keep-alive"
    },
    "RetryAttempts": 0
  }
}<jupyter_text>Create data set group in Amazon Personalize<jupyter_code>create_dataset_group_response = personalize.create_dataset_group(
    name = "cold-start-hotel-recommender-group-v2"
)

dataset_group_arn = create_dataset_group_response['datasetGroupArn']
print(json.dumps(create_dataset_group_response, indent=2))
max_time = time.time() + 3*60*60 # 3 hours
while time.time() < max_time:
    describe_dataset_group_response = personalize.describe_dataset_group(
        datasetGroupArn = dataset_group_arn
    )
    status = describe_dataset_group_response["datasetGroup"]["status"]
    print("DatasetGroup: {}".format(status))
    
    if status == "ACTIVE" or status == "CREATE FAILED":
        break
        
    time.sleep(60)<jupyter_output>DatasetGroup: CREATE PENDING
DatasetGroup: ACTIVE<jupyter_text>Create interaction data set in the group in Amazon Personalize<jupyter_code>create_dataset_interactions_response = personalize.create_dataset(
    name = "hotel-recommender-cold-start-dataset-v2",
    datasetType = "INTERACTIONS",
    datasetGroupArn = dataset_group_arn,
    schemaArn = schema_arn
)

dataset_interactions_arn = create_dataset_interactions_response['datasetArn']
print(json.dumps(create_dataset_interactions_response, indent=2))
role_arn = "arn:aws:iam::296654805457:role/Personalize"  # replace with a Role that has access to Personalize

create_dataset_interactions_import_job_response = personalize.create_dataset_import_job(
    jobName = "cold-start-hotel-recommender-dataset-import-job-v2",
    datasetArn = dataset_interactions_arn,
    dataSource = {
        "dataLocation": "s3://{}/{}".format(bucket, filename)
    },
    roleArn = role_arn
)

dataset_interactions_import_job_arn = create_dataset_interactions_import_job_response['datasetImportJobArn']
print(json.dumps(dataset_interactions_import_job_arn, indent=2))
max_time = time.time() + 3*60*60 # 3 hours
while time.time() < max_time:
    describe_dataset_import_job_response = personalize.describe_dataset_import_job(
        datasetImportJobArn = dataset_interactions_import_job_arn
    )
    status = describe_dataset_import_job_response["datasetImportJob"]['status']
    print("DatasetImportJob: {}".format(status))
    
    if status == "ACTIVE" or status == "CREATE FAILED":
        break
        
    time.sleep(60)<jupyter_output>DatasetImportJob: CREATE PENDING
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: ACTIVE<jupyter_text>Create item metadata data set in the group in Amazon Personalize<jupyter_code>create_dataset_item_metadata_response = personalize.create_dataset(
    name = "hotel-recommender-item-metadata-cold-start-dataset-v2",
    datasetType = "ITEMS",
    datasetGroupArn = dataset_group_arn,
    schemaArn = metadata_schema_arn
)

dataset_item_metadata_arn = create_dataset_item_metadata_response['datasetArn']
print(json.dumps(create_dataset_item_metadata_response, indent=2))
create_dataset_item_metatadata_import_job_response = personalize.create_dataset_import_job(
    jobName = "cold-start-hotel-recommender-item-metatadata-dataset-job-v2",
    datasetArn = dataset_item_metadata_arn,
    dataSource = {
        "dataLocation": "s3://{}/{}".format(bucket, filename_item_metadata)
    },
    roleArn = role_arn
)

dataset_item_metatadata_import_job_arn = create_dataset_item_metatadata_import_job_response['datasetImportJobArn']
print(json.dumps(dataset_item_metatadata_import_job_arn, indent=2))
max_time = time.time() + 3*60*60 # 3 hours
while time.time() < max_time:
    describe_dataset_import_job_response = personalize.describe_dataset_import_job(
        datasetImportJobArn = dataset_item_metatadata_import_job_arn
    )
    status = describe_dataset_import_job_response["datasetImportJob"]['status']
    print("DatasetImportJob: {}".format(status))
    
    if status == "ACTIVE" or status == "CREATE FAILED":
        break
        
    time.sleep(60)<jupyter_output>DatasetImportJob: CREATE PENDING
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: ACTIVE<jupyter_text>Create user metadata data set in the group in Amazon Personalize<jupyter_code>create_dataset_user_metadata_response = personalize.create_dataset(
    name = "hotel-recommender-user-metadata-cold-start-dataset-v2",
    datasetType = "USERS",
    datasetGroupArn = dataset_group_arn,
    schemaArn = user_metadata_schema_arn
)

dataset_user_metadata_arn = create_dataset_user_metadata_response['datasetArn']
print(json.dumps(create_dataset_user_metadata_response, indent=2))
create_dataset_user_metatadata_import_job_response = personalize.create_dataset_import_job(
    jobName = "cold-start-hotel-recommender-user-metatadata-dataset-import-v2",
    datasetArn = dataset_user_metadata_arn,
    dataSource = {
        "dataLocation": "s3://{}/{}".format(bucket, filename_user_metadata)
    },
    roleArn = role_arn
)

dataset_user_metatadata_import_job_arn = create_dataset_user_metatadata_import_job_response['datasetImportJobArn']
print(json.dumps(dataset_user_metatadata_import_job_arn, indent=2))
max_time = time.time() + 3*60*60 # 3 hours
while time.time() < max_time:
    describe_dataset_import_job_response = personalize.describe_dataset_import_job(
        datasetImportJobArn = dataset_user_metatadata_import_job_arn
    )
    status = describe_dataset_import_job_response["datasetImportJob"]['status']
    print("DatasetImportJob: {}".format(status))
    
    if status == "ACTIVE" or status == "CREATE FAILED":
        break
        
    time.sleep(60)<jupyter_output>DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: CREATE IN_PROGRESS
DatasetImportJob: ACTIVE<jupyter_text>List recommender algorithms/recipes available in Amazon Personalize<jupyter_code>list_recipes_response = personalize.list_recipes()
list_recipes_response
recipe_arn = "arn:aws:personalize:::recipe/aws-hrnn-metadata"<jupyter_output><empty_output><jupyter_text>Create solution in Amazon PersonalizeIn other words, let's train the hotel-recommender system!<jupyter_code>create_solution_response = personalize.create_solution(
    name = "hotel-recommender-cold-start-metadata-v2",
    datasetGroupArn = dataset_group_arn,
    recipeArn = recipe_arn
)

solution_arn = create_solution_response['solutionArn']
print(json.dumps(create_solution_response, indent=2))
create_solution_version_response = personalize.create_solution_version(
    solutionArn = solution_arn
)

solution_version_arn = create_solution_version_response['solutionVersionArn']
print(json.dumps(create_solution_version_response, indent=2))
max_time = time.time() + 3*60*60 # 3 hours
while time.time() < max_time:
    describe_solution_version_response = personalize.describe_solution_version(
        solutionVersionArn = solution_version_arn
    )
    status = describe_solution_version_response["solutionVersion"]["status"]
    print("SolutionVersion: {}".format(status))
    
    if status == "ACTIVE" or status == "CREATE FAILED":
        break
        
    time.sleep(60)
describe_solution_version_response["solutionVersion"]<jupyter_output><empty_output><jupyter_text>Time to retrieve accuracy metrics of the trained recommender system model!<jupyter_code>get_solution_metrics_response = personalize.get_solution_metrics(
    solutionVersionArn = solution_version_arn
)

print(json.dumps(get_solution_metrics_response, indent=2))<jupyter_output>{
  "solutionVersionArn": "arn:aws:personalize:us-east-1:296654805457:solution/hotel-recommender-cold-start-metadata-v2/2cc183a8",
  "metrics": {
    "coverage": 0.9901,
    "mean_reciprocal_rank_at_25": 0.4308,
    "normalized_discounted_cumulative_gain_at_10": 0.5153,
    "normalized_discounted_cumulative_gain_at_25": 0.5558,
    "normalized_discounted_cumulative_gain_at_5": 0.481,
    "precision_at_10": 0.0636,
    "precision_at_25": 0.0319,
    "precision_at_5": 0.1071
  },
  "ResponseMetadata": {
    "RequestId": "70a6de50-897f-4bac-af6d-e4770be34b53",
    "HTTPStatusCode": 200,
    "HTTPHeaders": {
      "content-type": "application/x-amz-json-1.1",
      "date": "Mon, 25 Nov 2019 13:30:05 GMT",
      "x-amzn-requestid": "70a6de50-897f-4bac-af6d-e4770be34b53",
      "content-length": "423",
      "connection": "keep-alive"
    },
    "RetryAttempts": 0
  }
}<jupyter_text>Let's deploy the trained recommender model!<jupyter_code>create_campaign_response = personalize.create_campaign(
    name = "hotel-recommender-cold-start-metadata-campaign-v1",
    solutionVersionArn = solution_version_arn,
    minProvisionedTPS = 1
)

campaign_arn = create_campaign_response['campaignArn']
print(json.dumps(create_campaign_response, indent=2))
max_time = time.time() + 3*60*60 # 3 hours
while time.time() < max_time:
    describe_campaign_response = personalize.describe_campaign(
        campaignArn = campaign_arn
    )
    status = describe_campaign_response["campaign"]["status"]
    print("Campaign: {}".format(status))
    
    if status == "ACTIVE" or status == "CREATE FAILED":
        break
        
    time.sleep(60)
train_df[train_df['user_id'] == 194297]
test_df[test_df['user_id'] == 194297]
get_recommendations_response = personalize_runtime.get_recommendations(
    campaignArn = campaign_arn,
    userId = '194297'
)
get_recommendations_response
train_df[train_df['user_id'] == 837487]
test_df[test_df['user_id'] == 837487]
personalize_runtime.get_recommendations(
    campaignArn = campaign_arn,
    userId = '837487'
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10791407896985927, 'fraction_numerical': 0.021423280724745027, 'mean_word_length': 5.19774011299435, 'pattern_counts': {'":': 87, '<': 61, '<?xml version=': 0, '>': 55, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '170770', 'license': 'MIT', 'n_tokens_mistral': 6135, 'n_tokens_neox': 5821, 'n_words': 999, 'path': 'hotel-recommender-meta-data-personalize.ipynb', 'repo_name': 'pm3310/reinvent-2019-hotel-recommender', 'size': '17551'}