<jupyter_start><jupyter_text>Load the Dataset<jupyter_code>%cd /content/Peper-Analysis
sofia_airbnb = pd.read_csv("./data/sofia_airbnb_reviews.csv")
sofia_airbnb.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 306 entries, 0 to 305
Data columns (total 5 columns):
 #   Column          Non-Null Count  Dtype  
---  ------          --------------  -----  
 0   lat             306 non-null    float64
 1   long            306 non-null    float64
 2   reviews         306 non-null    object 
 3   sentiments      306 non-null    object 
 4   mean_sentiment  306 non-null    float64
dtypes: float64(3), object(2)
memory usage: 12.1+ KB<jupyter_text>***  Quick stats<jupyter_code>sofia_airbnb.head()
sofia_airbnb.describe()<jupyter_output><empty_output><jupyter_text>***  Exploring the reviews  Remove NaN and redundant reviews<jupyter_code>sofia_airbnb.dropna(inplace=True)
sofia_airbnb.reset_index(inplace=True)
sofia_airbnb.drop("index", axis=1, inplace=True)
for i in range(sofia_airbnb.shape[0]):
    try:
        json.loads(sofia_airbnb["reviews"][i])
    except json.JSONDecodeError:
        sofia_airbnb.drop(i, axis=0, inplace=True)<jupyter_output><empty_output><jupyter_text>Sentiment Analysis  Quaintitative<jupyter_code>sentiments = []
mean_sentiment = []

for index, loc in sofia_airbnb.iterrows():
    scores = []
    for review in json.loads(loc["reviews"]):
        scores.append(get_sentiment(review))     
        
    sentiments.append(scores)
    mean_sentiment.append(np.mean(scores))
    
sofia_airbnb["sentiments"] = sentiments
sofia_airbnb["mean_sentiment"] = mean_sentiment
sns.distplot(sofia_airbnb["mean_sentiment"], kde=False);<jupyter_output>/usr/local/lib/python3.7/dist-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)<jupyter_text>Save the `mean_sentiment`s to `JSON`<jupyter_code>jsonArray = ''
for index, row in sofia_airbnb.iterrows():
    x = {
       "lat": row["lat"],
       "long": row["long"],
       "mean_sentiment": row["mean_sentiment"]
    }
    jsonArray += json.dumps(x)+','<jupyter_output><empty_output><jupyter_text>Topic Modelling<jupyter_code>topics = []
import nltk
nltk.download('punkt')
nltk.download('wordnet')
for index, loc in sofia_airbnb.iterrows():
    current_loc_topics = []
    for review in json.loads(loc["reviews"]):
        try:
            current_loc_topics.append(' '.join(TopicModelling(review).get_topics()[0]))
        except ValueError:
            pass
        
    topics.append(current_loc_topics)
        
sofia_airbnb["topics"] = topics<jupyter_output>[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Unzipping corpora/wordnet.zip.<jupyter_text>Term-Frequency x Inverse-Doc-Freq<jupyter_code>for index, loc in sofia_airbnb.iterrows():
    tf_idf = TfidfVectorizer(use_idf=True, stop_words=stop_words_en)
    X = tf_idf.fit_transform(json.loads(loc["reviews"]))
    
    tfs = np.mean(X.todense(), axis=0) * .1
    scores = np.multiply(tfs, tf_idf.idf_.reshape(tfs.shape))
    
    terms = np.array(tf_idf.get_feature_names()).reshape(tfs.shape)
    zipped = np.hstack((terms.T, scores.T)).tolist()
    print(np.array(sorted(zipped, key=lambda x: x[1], reverse=True)))
    
    print('='*100)
    print('='*100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1331871745683749, 'fraction_numerical': 0.01644286105782406, 'mean_word_length': 4.0835654596100275, 'pattern_counts': {'":': 3, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '370708', 'license': 'MIT', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1343, 'n_words': 232, 'path': 'data-analysis/Sofia_Airbnb_Analyse.ipynb', 'repo_name': 'So-AI-love/Peper-Analysis', 'size': '3649'}