<jupyter_start><jupyter_text>Patodeweloperka EDA<jupyter_code>df_small = df.loc[(df['flat_area'] < 27) & (10 < df['flat_area'])]
df_small['price_per_m2'] = df_small['price']/df_small['flat_area']
len(df_small)
plt.hist(df_small['price'])
ax = plt.gca()
ax.get_xaxis().get_major_formatter().set_scientific(False)
plt.show()
plt.hist(df_small['price_per_m2'])
ax = plt.gca()
ax.get_xaxis().get_major_formatter().set_scientific(False)
plt.show()
from collections import Counter
from spacy.lang.pl.stop_words import STOP_WORDS
small_descriptions = []
for desc in df_small['description']:
    desc = [x.lower() for x  in desc.split()]
    small_descriptions +=  desc

to_replace = r'0123456789.,/\:;-=!@#$%^&*()_+[]{}|'

desc_clean = []

for token in small_descriptions:
    for repl in list(to_replace):
        token = token.replace(repl, ' ').strip()

    if token not in STOP_WORDS:
        desc_clean.append(token)

c = Counter(desc_clean)
# print(c.most_common)<jupyter_output><empty_output><jupyter_text>NLP<jupyter_code>import spacy
import pl_core_news_lg

nlp = pl_core_news_lg.load()


from sklearn.model_selection import train_test_split


train_set, test_set= train_test_split(df, test_size=0.2, random_state=42)
# Decision Tree baseline
# KNN Regression
# Random Forest Regression

print(train_set.head())
from sklearn.tree import DecisionTreeRegressor

regressor = DecisionTreeRegressor(random_state=0)

regressor.fit(train_set, test_set)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14014895057549087, 'fraction_numerical': 0.014218009478672985, 'mean_word_length': 6.50253807106599, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '294429', 'license': 'Apache-2.0', 'n_tokens_mistral': 589, 'n_tokens_neox': 556, 'n_words': 94, 'path': 'src/eda_nlp.ipynb', 'repo_name': 'mbalcerzak/warsaw-real-estate-analysis', 'size': '1477'}