<jupyter_start><jupyter_text>Loading Data  Rating and reviews<jupyter_code>rr_cosmetic_01_20 = pd.read_csv("../data/Rating and Reviews/cosmetics_reviews_20200101-20200131.csv")
rr_skincare_12_19 = pd.read_csv("../data/Rating and Reviews/skincare_reviews_20191231.csv")
rr_skincare_11_19 = pd.read_csv("../data/Rating and Reviews/skincare_reviews_20191130.csv")<jupyter_output>C:\Users\asaid\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3063: DtypeWarning: Columns (27,28,29,32,33,37) have mixed types.Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)
C:\Users\asaid\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3063: DtypeWarning: Columns (33,34) have mixed types.Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)
C:\Users\asaid\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3063: DtypeWarning: Columns (1,23,24,25,26,27,28,29,30,32,33,34,35,36,37,38,39) have mixed types.Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>Product Catalogue<jupyter_code>pc_cosmetic_01_20 = pd.read_csv("../data/Product Catalogue/Cosmetics_Product_20200116.csv")
pc_skincare_01_20 = pd.read_csv("../data/Product Catalogue/SkinCare_Product_20200116.csv")<jupyter_output>C:\Users\asaid\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3063: DtypeWarning: Columns (16,17) have mixed types.Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>Overview<jupyter_code>rr_cosmetic_01_20.describe()
rr_skincare_12_19.describe()<jupyter_output><empty_output><jupyter_text>Difference between the two datasets  Common columns<jupyter_code>common_cols = set(rr_skincare_11_19.columns).intersection(set(rr_cosmetic_01_20.columns))
print("Common columns between the two datasets:")
for col in common_cols:
    print(col)<jupyter_output>Common columns between the two datasets:
Verified Reviewer
Ingredient
Feature
Skin Tone
Product_ID
Sentiment
Source Product Identifier
Cons
Syndication Source
Geography
Benefit
Description
From
Additional Ingredients (no rulebase)
Title
Age
OnlineStatement_ID
Reviewer Skin Type
I shop at macys.com
Hair Color
Best For
Product Form
Rating
Purchase Location
Channel
Describe Yourself
Date
ELC Solution Type
Verified Buyer
Recommended
Eye Color
OnlinePost_ID
Pros
Make-up Style
Gender<jupyter_text>Different columns<jupyter_code>common_cols = set(rr_skincare_11_19.columns).difference(set(rr_cosmetic_01_20.columns))
for col in common_cols:
    if col in rr_cosmetic_01_20:
        print('Column {:15} belongs to Cosmetic but not to Skincare'.format(col))
    else:
        print('Column {:15} belongs to Skincare but not to Cosmetic'.format(col))<jupyter_output>Column Skin Condition  belongs to Skincare but not to Cosmetic
Column Use Case        belongs to Skincare but not to Cosmetic
Column Treatment Area  belongs to Skincare but not to Cosmetic
Column Packaging       belongs to Skincare but not to Cosmetic
Column Skin Type       belongs to Skincare but not to Cosmetic<jupyter_text>Merging the two datasets<jupyter_code>rr_cosmetic_01_20['Dataset'] = 'Cosmetic'
rr_skincare_11_19['Dataset'] = 'Skincare'
df = pd.concat([rr_cosmetic_01_20, rr_skincare_11_19])
df.head()
df['Syndication Source'].unique()
df.describe()<jupyter_output><empty_output><jupyter_text>Geography<jupyter_code>df.groupby(['Dataset', 'Geography']).size().unstack(0)
df.groupby(['Dataset', 'Geography']).size().unstack(0).plot.bar(figsize=[8,5])
plt.grid()
plt.title('Geography')
plt.xlabel('Number of ratings')
plt.ylabel('Geography')
df=df[df['Geography']=='USA']<jupyter_output><empty_output><jupyter_text>Dataset sizes<jupyter_code>df.groupby('Dataset').size()
df.groupby('Dataset').size().plot.bar(figsize=[5,5])
plt.grid()
plt.title('Size of datasets')
plt.ylabel('Number of rows')
plt.xlabel('Dataset')<jupyter_output><empty_output><jupyter_text>Number of unique values<jupyter_code>print('cos', len(rr_cosmetic_01_20.columns))
print('skin', len(rr_skincare_12_19.columns))
nb_unique = df.copy().groupby('Dataset').nunique().stack().unstack(0)
nb_unique
nb_unique.plot.bar()
plt.grid()
plt.xlabel('Column')
plt.ylabel('Number of unique values')<jupyter_output><empty_output><jupyter_text>Missing Data<jupyter_code>df_pct_na = (df.set_index('Dataset').isna().reset_index().groupby('Dataset').sum().divide(df.groupby('Dataset').size(), axis=0)*100).stack().unstack(0)
df_pct_na
len(pc_skincare_01_20.columns)
df_pct_na.plot.bar(figsize=[10,5])
plt.grid()
plt.title('Missing data per dataset')
plt.ylabel('% of  NaN')
plt.xlabel('Column')<jupyter_output><empty_output><jupyter_text>Ratings<jupyter_code>df[['Dataset', 'Rating']].boxplot(by='Dataset', figsize=[6,5])
plt.xlabel('Dataset')
plt.ylabel('Rating')
df.hist('Rating', by='Dataset', grid=True, bins=50)<jupyter_output><empty_output><jupyter_text>Rating by channel<jupyter_code>df.groupby('Channel')['Rating'].describe()
df['Rating'] = df['Rating'].astype(int)
ratings_channel = (df.groupby(['Channel','Rating'])['Rating'].count().divide(df.groupby('Channel')['Rating'].count())*100)
ratings_channel.unstack(0).plot.bar(figsize=[8,5])
plt.xlabel('Rating')
plt.ylabel('% of total ratings')
plt.title('Distribution of ratings by Channel')
plt.grid()
df.hist('Rating', by='Channel', grid=True, bins=50, figsize=[10,10])
df[['Channel', 'Rating']].boxplot(by='Channel', figsize=[6,5], rot=90)
plt.xlabel('Channel')
plt.ylabel('Rating')<jupyter_output><empty_output><jupyter_text>Number of reviews by channel<jupyter_code>(df.groupby(['Dataset','Channel']).size().divide(df.groupby('Dataset').size(), axis=0)*100).unstack(0).plot.barh(figsize=[8,5])
plt.grid()
plt.xlabel('% of total reviews')
plt.title('Distribution of reviews by channel')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14213962832747362, 'fraction_numerical': 0.040683073832245106, 'mean_word_length': 8.604501607717042, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '324242', 'license': 'MIT', 'n_tokens_mistral': 2315, 'n_tokens_neox': 2152, 'n_words': 363, 'path': 'ratings_and_reviews_wrangling/old_files/rating_and_reviews_data_exploration.ipynb', 'repo_name': 'Anis-Bensaid/CapstoneProject', 'size': '5973'}