<jupyter_start><jupyter_text>Outlier, huge basement, low price. let's remove it<jupyter_code>fig, ax = plt.subplots()
ax.scatter(x = df['TotalBsmtSF'], y = df['SalePrice'])
plt.ylabel('SalePrice', fontsize=13)
plt.xlabel('TotalBsmtSF', fontsize=13)
plt.show()
# delete outliers
df = df.drop(df[(df['TotalBsmtSF']>5000) & (df['SalePrice']<300000)].index) # basement>5k and price under 300k

# reset index
df = df.reset_index(drop=True)

# check the graphic again
fig, ax = plt.subplots()
ax.scatter(df['TotalBsmtSF'], df['SalePrice'])
plt.ylabel('SalePrice', fontsize=13)
plt.xlabel('TotalBsmtSF', fontsize=13)
plt.show()<jupyter_output><empty_output><jupyter_text>let's check GrLivingArea<jupyter_code>fig, ax = plt.subplots()
ax.scatter(x = df['GrLivArea'], y = df['SalePrice'])
plt.ylabel('SalePrice', fontsize=13)
plt.xlabel('GrLivArea', fontsize=13)
plt.show()
df = df.drop(df[(df['GrLivArea']>4000) & (df['SalePrice']<300000)].index) #AREA>4k & price<300k DROPPED
df = df.reset_index(drop=True)

fig, ax = plt.subplots()
ax.scatter(df['GrLivArea'], df['SalePrice'])
plt.ylabel('SalePrice', fontsize=13)
plt.xlabel('GrLivArea', fontsize=13)
plt.show() # Nice and linear. Those top two were NOT outliers, they're in the line.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1836248012718601, 'fraction_numerical': 0.034976152623211444, 'mean_word_length': 7.804195804195804, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '364594', 'license': 'MIT', 'n_tokens_mistral': 558, 'n_tokens_neox': 506, 'n_words': 91, 'path': 'w3/w3-day_3/Outlier_detection.ipynb', 'repo_name': 'bmskarate/lighthouseMain', 'size': '1258'}