<jupyter_start><jupyter_text>The data consists of 10 fields:- *Points*: the number of points WineEnthusiast rated the wine on a scale of 1-100 (though they say they only post reviews for wines that score >=80)- *Title*: the title of the wine review, which often contains the vintage if you're interested in extracting that feature- *Variety*: the type of grapes used to make the wine (ie Pinot Noir)- *Description*: a few sentences from a sommelier describing the wine's taste, smell, look, feel, etc.- *Country*: the country that the wine is from- *Province*: the province or state that the wine is from- *Region 1*: the wine growing area in a province or state (ie Napa)- *Region 2*: sometimes there are more specific regions specified within a wine growing area (ie Rutherford inside the Napa Valley), but this value can sometimes be blank- *Winery*: the winery that made the wine- *Designation*: the vineyard within the winery where the grapes that made the wine are from- *Price*: the cost for a bottle of the wine - *Taster Name*: name of the person who tasted and reviewed the wine- *Taster Twitter Handle*: Twitter handle for the person who tasted ane reviewed the wine<jupyter_code>review.info() # (150930, 10)
total_sample = len(review.index)
review.describe()
# Features has missing values
feat = list()
missing_num = list()

for col in review.columns:
    if True in review[col].isnull().values:
        feat.append(col)
        number = review[col].isnull().sum()
        missing_num.append(number)
        print(col, 'has {} missing value'.format(number), \
              ', with {0:.4f}% missing rate'.format(number / total_sample * 100))

x, y = feat, missing_num
plt.rcParams.update({'font.size': 15})
plt.figure(figsize=(9, 5))
plt.bar(x, y, alpha=1)
plt.xlabel('Features', fontsize=15)
plt.ylabel('Missing number', fontsize=15)
plt.title('Missing Value')
# Descriptiom
for i in range(len(review.index)):
    review.loc[i, 'description'] = len(review.loc[i, 'description'])
review.head()
# Country
countries = review.country.unique() 
countries
review.country.isnull().sum()
review.loc[pd.isnull(review.country)]
# encoding Country
print('Country number:', len(countries))
country_dict = dict()
review.loc[pd.isnull(review['country']), ['country']] = 'Beijing'
# null_ID = review[pd.isnull(review['country'])].index
# print(null_ID)
# notnull_ID = list(set(review.index) - set(null_ID))
# tmp_review = review.iloc[notnull_ID]
# countries = tmp_review['country'].unique()
countries = review.country.unique()
for i in range(len(countries)):
    country_dict[countries[i]] = i

for c in countries:
    review['country'] = review['country'].replace(c, country_dict[c])

review.head()
# review.loc[review['country'] == -1]
# Designation
des = review.designation.unique()
len(des)
review.designation.isnull().sum()
review.loc[pd.isnull(review.designation)].head()
# encoding designation
null_ID = review[pd.isnull(review['designation'])].index
notnull_ID = list(set(review.index) - set(null_ID))
des = review.iloc[notnull_ID]
des = des['designation'].unique()
# print(des)
des_dict = dict()
for i in range(len(des)):
    des_dict[des[i]] = i
# print(des_dict)
for d in des:
    review['designation'] = review['designation'].replace(d, des_dict[d])
review.loc[null_ID, ['designation']] = -1
review.head(10)
# Points
p = review.points.unique()
p # no missing point
(review['points'] == 100).sum()
review['points'].describe()
plt.figure(figsize=(7, 7))
plt.hist(review['points'],  density=False, alpha=1,
        histtype='bar', color='steelblue', 
        edgecolor='k')
plt.xlabel('Points')
plt.ylabel('Number')
plt.title('Point Distribution')
# encoding points
point_mean = review['points'].mean()
review.loc[(review['points'] > 96) & (review['points'] <= 100), 'points'] = 1
review.loc[(review['points'] > 92) & (review['points'] <= 96), 'points'] = 2
review.loc[(review['points'] > 88) & (review['points'] <= 92), 'points'] = 3
review.loc[(review['points'] > 84) & (review['points'] <= 88), 'points'] = 4
review.loc[(review['points'] >= 80) & (review['points'] <= 84), 'points'] = 5
review.head(10)
# Price
price_counts = review.price.value_counts()
price_counts.head(10)
pd.isnull(review.price).sum()
review['price'].unique()[: 40]
review['price'].describe()
review.loc[pd.isnull(review.price)].head()
review['price'] = review['price'].apply(lambda x: -20.0 if pd.isnull(x) else x)

plt.figure(figsize=(9, 6))
plt.hist(review['price'], alpha=1, color='r',
         histtype='bar', density=False,
         bins=1000)
plt.axis([-30, 280, 0, 16000])
plt.xlabel('Price')
plt.ylabel('Number')
plt.title('Price Distribution')
# list(review[review['price'] > 200].index)
# encoding price
# min(except -20): 4, max: 2300
null_ID = review[pd.isnull(review['price'])].index
review.loc[null_ID, ['price']] = -1
# notnull_ID = list(set(review.index) - set(null_ID))
# temp_review = review.iloc[notnull_ID]
# srt = sorted(set(temp_review['price'].values))
# interval = np.ceil((srt[-1] - srt[1]) / 100) # 23
price = review['price'].unique()
# print(price)
for i in price:
    review['price'] = review['price'].replace(i, np.floor(i / 100) + 1)

review.head(10)
# Province
prov = review.province.unique()
len(prov)
pd.isnull(review.province).sum()
prov[: 40]
review.loc[pd.isnull(review.province)]
# encoding province
review.loc[pd.isnull(review['province']), ['province']] = 'Beijing'

"""
prov_dict = dict()
for p in range(len(prov)):
    prov_dict[prov[p]] = p
for p in prov:
    review['province'] = review['province'].replace(p, prov_dict[p])
review.head()
"""
review.loc[review['province'] == 'Beijing']
# review.to_csv('./data/temp.csv')
# region_1
region1 = review.region_1.unique()
region1
len(region1)
review.region_1.isnull().sum()
review.loc[pd.isnull(review['region_1'])].head()
# region_2
region2 = review['region_2'].unique()
region2
len(region2)
pd.isnull(review['region_2']).sum()
review.loc[review['region_2'].isnull()].head()
# merge region_1 and region_2 to new feature region
review.loc[pd.isnull(review['region_2']), ['region_2']] = ''
review.loc[pd.isnull(review['region_1']), ['region_1']] = ''
review.loc[:, 'region'] = review.loc[:, 'province'] + ' ' + review.loc[:, 'region_1'] + ' ' + review.loc[:, 'region_2']
# review.loc[:, 'region'] = review.loc[:, 'region_1'] + ' ' + review.loc[:, 'region_2']
review.head()
# encoding region
"""
null_ID = review[pd.isnull(review['region_1'])].index
new_ID = list(set(review.index) - set(null_ID))
review = review.iloc[new_ID]
"""
review.region.head()
region = review['region'].unique()
print(len(region))
region_dict = dict()
for r in range(len(region)):
    region_dict[region[r]] = r
for reg in region:
    review['region'] = review['region'].replace(reg, region_dict[reg])
    # review.loc[review['region'] == reg, ['region']] = region_dict[reg]
review.head()
# Variety 
variety = review['variety'].unique()
len(variety)
variety[: 20]
# encoding variety
variety_dict = dict()
for v in range(len(variety)):
    variety_dict[variety[v]] = v
for var in variety:
    review['variety'] = review['variety'].replace(var, variety_dict[var])
review.head()
# Winery
winery = review['winery'].unique()
len(winery)
winery[: 20]
# encoding winery
winery_dict = dict()
for w in range(len(winery)):
    winery_dict[winery[w]] = w
for win in winery:
    review['winery'] = review['winery'].replace(win, winery_dict[win])
review.head()
review.shape
review.to_csv('./data/wine_vector_2.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18167224080267558, 'fraction_numerical': 0.02020066889632107, 'mean_word_length': 5.506527415143603, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '518997', 'license': 'MIT', 'n_tokens_mistral': 2644, 'n_tokens_neox': 2519, 'n_words': 624, 'path': 'data_process.ipynb', 'repo_name': 'fossabot/wine_review', 'size': '7475'}