<jupyter_start><jupyter_text>Packages<jupyter_code>import numpy as np
import sys
import os
import pandas as pd
import boto3

# Data storage
from sqlalchemy import create_engine  # SQL helper
import psycopg2 as psql  # PostgreSQL DBs

sys.path.append("..")
import keys
import data_fcns as dfc<jupyter_output><empty_output><jupyter_text>Options<jupyter_code># So that items in pandas columns don't show truncated values 
pd.set_option('display.max_colwidth', -1)<jupyter_output><empty_output><jupyter_text>Create Official item factors matrix or dataframe<jupyter_code>item_factors_df = pd.read_pickle('support_data/item_factors.pkl')
item_factors_df.head()<jupyter_output><empty_output><jupyter_text>Get comics data from DB<jupyter_code># Define path to secret
secret_path_aws = os.path.join(os.environ['HOME'], '.secret',
                               'aws_ps_flatiron.json')
secret_path_aws

aws_keys = keys.get_keys(secret_path_aws)
user = aws_keys['user']
ps = aws_keys['password']
host = aws_keys['host']
db = aws_keys['db_name']

aws_ps_engine = ('postgresql://' + user + ':' + ps + '@' + host + '/' + db)

# Setup PSQL connection
conn = psql.connect(
    database=db,
    user=user,
    password=ps,
    host=host,
    port='5432'
)
# Instantiate cursor
cur = conn.cursor()
#  Count records.
query = """
    SELECT * FROM comics 
"""

# Execute the query
cur.execute(query)

conn.commit()
# Check results
comics_pdf = pd.DataFrame(cur.fetchall())
comics_pdf.columns = [col.name for col in cur.description]
comics_pdf.head()<jupyter_output><empty_output><jupyter_text>Merge item factors and comics<jupyter_code># Change column names for item_factors_df
item_factors_df.columns = ['comic_id', 'features']
combo = item_factors_df.merge(comics_pdf, left_on='comic_id', right_on='comic_id', how='inner', )
combo.set_index(['comic_id'], inplace=True)
combo.head()
comics_pdf.loc[comics_pdf['comic_id']==20]
combo.shape
combo.drop(['img_url'], axis=1, inplace=True)
combo.shape
combo.head()
combo.columns = ['features', 'comic_title', 'img_url']
combo.head()<jupyter_output><empty_output><jupyter_text>Create pickle file<jupyter_code>combo.to_pickle('support_data/comics_factors_201908.pkl')<jupyter_output><empty_output><jupyter_text>Test 2019.08.12 fix<jupyter_code># Read new pickle
cf_new = pd.read_pickle('support_data/comics_factors_201908.pkl')
# Read old pickle 
cf_old = pd.read_pickle('support_data/comics_factors.pkl')
cf_new.shape
cf_old.shape
cf_new.head()
cf_old.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12977707006369427, 'fraction_numerical': 0.01194267515923567, 'mean_word_length': 6.456973293768546, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '497247', 'license': 'MIT', 'n_tokens_mistral': 971, 'n_tokens_neox': 937, 'n_words': 170, 'path': 'comrx/dev/deprecated/comics_rx-03c_build_comics_factors_v1.ipynb', 'repo_name': 'MangrobanGit/comic_rx', 'size': '2512'}