<jupyter_start><jupyter_text>Below code is inspired by: https://www.kaggle.com/brittabettendorf/nlp-on-airbnb-data/notebook DATA VIEWING<jupyter_code>#read the 'reviews_summary.csv' file with data
df_reviews_summary = pd.read_csv('data/reviews_summary.csv')
df_reviews_summary.shape
#checking the duplicates
df_reviews_summary.duplicated().sum()
df_reviews_summary.sample(20)
#read the 'listings_summary.csv' file with data
df_listings_summary = pd.read_csv('data/listings_summary.csv')
df_listings_summary.shape
df_listings_summary.sample(10)
# merging full reviews + add only specific columns from listings dataframe
df = pd.merge(df_reviews_summary, df_listings_summary[['neighbourhood_group_cleansed', 'host_id', 'latitude',
                          'longitude', 'number_of_reviews', 'id', 'property_type']], 
              left_on='listing_id', right_on='id', how='left')

df.rename(columns = {'id_x':'id', 'neighbourhood_group_cleansed':'neighbourhood_group'}, inplace=True)
df.drop(['id_y'], axis=1, inplace=True)
df.sample(10)
#group by host_id and count the number of flats for each host 
flats_per_host = pd.DataFrame(df.groupby('host_id')['listing_id'].nunique())
#top 20 sorted
flats_per_host.sort_values(by=['listing_id'], ascending=False, inplace=True)
flats_per_host.head(20)
def check_top_host_neighbourhood(df, host_id):
    top_host = df['host_id'] == host_id
    df[top_host]['neighbourhood_group'].value_counts()
    print ('Host number: '+ str(host_id))
    return pd.DataFrame(df[top_host].groupby('neighbourhood_group')['listing_id'].nunique())    
def check_top_host_property(df, host_id):
    top_host = df['host_id'] == host_id
    df[top_host]['neighbourhood_group'].value_counts()
    print ('Host number: '+ str(host_id))
    return pd.DataFrame(df[top_host].groupby('property_type')['listing_id'].nunique())
#checking the no. 1 host, he has 44 apartments in 8 different districts
check_top_host_neighbourhood(df, flats_per_host.index[0])
check_top_host_property(df, flats_per_host.index[0])
#checking the no. 2 host, turned out to be a hostel
check_top_host_neighbourhood(df, flats_per_host.index[1])
check_top_host_property(df, flats_per_host.index[1])
#checking the no. 3 host, different properties in two districts
check_top_host_neighbourhood(df, flats_per_host.index[2])
check_top_host_property(df, flats_per_host.index[2])<jupyter_output>Host number: 2293972<jupyter_text>DATA PREPARATION<jupyter_code>#missing values, 496 comments are missing
df.isna().sum()
#496 is not so much so I delete those data
df.dropna(inplace=True)
df.isna().sum()<jupyter_output><empty_output><jupyter_text>LANGUAGE DETECTION<jupyter_code>#function to detect language of comment
def language_detection(text):
    try:
        return detect(text)
    except:
        return None
%%time
df['language'] = df['comments'].apply(language_detection)
#write the dataframe to a csv file in order to avoid the long runtime
df.to_csv('prepared_df', index=False)
df = pd.read_csv('prepared_df')
df.sample(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1242603550295858, 'fraction_numerical': 0.012491781722550954, 'mean_word_length': 6.842783505154639, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '538758', 'license': 'MIT', 'n_tokens_mistral': 1122, 'n_tokens_neox': 1064, 'n_words': 211, 'path': 'German Sentiment Analysis - AirBNB example.ipynb', 'repo_name': 'andrzej-malina/German-Sentiment-Analysis---AirBNB-example', 'size': '3042'}