<jupyter_start><jupyter_text>i already filtered the cars with 0dt price tags and >=200 000dt because i noticed people put their phone number in there tooto do:    -normalize Kms_driven numbers , and price tag ,some people refer to 10 000dt as 10dt    -convert year to integer    -add a car age column<jupyter_code>df[['city','region']] = df.location.str.split(', ',expand=True)
df.drop('location',axis=1, inplace=True)
df.head()
#check for null values
df.isnull().sum()
df.dropna()

#df['year']=df['year'].astype(int)
#df.drop(df[df['year']=='BMW'].index,inplace=True)
df = df[~df['year'].isnull()]

df['year']=df['year'].astype(int)
df.dtypes
#handling year column:
df.year.describe()
#removing faulty years : cars that were made before 1800 and after 2021
absurd_year_past=df.year > 2021
absurd_year_future=df.year < 1800
print(absurd_year_past)
print(absurd_year_future)
absurd_year_past.unique()
absurd_year_future.unique()
absurd_year=absurd_year_past|absurd_year_future
df.drop(df[absurd_year].index,inplace=True)

df['Current Year']=2021
df['nb_year']=df['Current Year']- df['year']
df=df.drop(['Current Year'],axis=1)

df.head()
df.describe(include='all')
#handling Kms_driven

df.Kms_Driven.describe()

#as u can see we have negative values
drop_negatives=df.Kms_Driven < 0
df.drop(df[drop_negatives].index, inplace=True)
df.Kms_Driven.describe()
#handling prices
df.price.describe()
#some people refer to 10 000dt as 10dt
below100 = df.price < 100
df.loc[below100,'price'] *= 1000
df.price.describe(include="all")
import seaborn as sns
sns.pairplot(df)
#date column serves no perpose
df.drop(columns=['date'],inplace=True)
#handling horse power column

outliers= df.loc[~df['Horse_pow'].astype(str).str.isdigit(), 'Horse_pow'].tolist() #finding the outlaws that aren't a null value
print(outliers)
df = df[~df['Horse_pow'].isin(outlaws)]#removing outlaws from Horse power Column

df = df[~df['Horse_pow'].isnull()]#removing nulled value from Horse Pow Column
df['Horse_pow']=df['Horse_pow'].astype(int)#converting Horse Power Column Type to integer
df.Horse_pow.describe()
# 0 horse power is not logical so we're gonna assign the mean value to them
zeroHP = df.Horse_pow == 0.0

df.loc[zeroHP,'Horse_pow']= df.Horse_pow.mean()
df.Horse_pow.describe()
df.Fuel_Type.unique()
#region is filled with Autre villes , so we're going to remove it and only keep the city column
df.drop(["region"],axis=1,inplace=True)

#making sure we have all the cities: 24
print(df.city.unique())

len(df.city.unique())
df.Color.unique()
df.drop(["Color"],axis=1,inplace=True)

df.brand.unique()
df.model.nunique()
df.isnull().sum()
df = df[~df['Kms_Driven'].isnull()]
df = df[~df['price'].isnull()]

df.isnull().sum()
df.body_style.unique()
df['body_style'].describe()
df.to_csv('clean_tayara_cars_dataset.csv')
data=pd.read_csv('clean_tayara_cars.csv')
polo=data.loc[data['model']=='Polo']
polo.boite.unique()
polo.cylindre.unique()
data.drop(["body_style"],axis=1,inplace=True)

#data.drop(["Unnamed: 0"],axis=1,inplace=True)
data=data[data['Horse_pow']<20]

data.head()

data.to_csv("clean_tayara_cars.csv")
data = data.groupby('brand').filter(lambda x: len(x) > 50)
data['brand'].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1644900497512438, 'fraction_numerical': 0.022077114427860697, 'mean_word_length': 6.884803921568627, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '379006', 'license': 'MIT', 'n_tokens_mistral': 1287, 'n_tokens_neox': 1199, 'n_words': 263, 'path': 'DataCleaning_Cars.ipynb', 'repo_name': 'Medsabkhi21/tayara_cars_price_prediction', 'size': '3216'}