<jupyter_start><jupyter_text>Index problem finxing and data cleaning<jupyter_code>avocado_2015_to_2017 = avocado_2015_to_2017.sort_values("Date")
avocado_2015_to_2017.reset_index(inplace=True)
del avocado_2015_to_2017['index']
avocado_2015_to_2017 = avocado_2015_to_2017[avocado_2015_to_2017.year != 2018]
avocado_2018.drop(['Timeframe'], inplace=True,  axis = 1)
avocado_2019.drop(['Timeframe'], inplace=True,  axis = 1)
avocado_2020.drop(['Timeframe'], inplace=True,  axis = 1)<jupyter_output><empty_output><jupyter_text>Renaming Columns<jupyter_code>avocado_2015_to_2017.rename(columns = {'region':'Region' , 'type':'Type' , 'year':'Year'} , inplace = True)
avocado_2018.rename(columns = {'Geography':'Region', 'Current Year Week Ending':'Date', 'Type':'Type', 'ASP Current Year':'AveragePrice',
       'Total Bulk and Bags Units':'Total Volume', '4046 Units':'4046', '4225 Units':'4225',
       '4770 Units':'4770', 'TotalBagged Units':'Total Bags', 'SmlBagged Units':'Small Bags',
       'LrgBagged Units':'Large Bags', 'X-LrgBagged Units':'XLarge Bags'}, inplace = True)
avocado_2019.rename(columns = {'Geography':'Region', 'Current Year Week Ending':'Date', 'Type':'Type', 'ASP Current Year':'AveragePrice',
       'Total Bulk and Bags Units':'Total Volume', '4046 Units':'4046', '4225 Units':'4225',
       '4770 Units':'4770', 'TotalBagged Units':'Total Bags', 'SmlBagged Units':'Small Bags',
       'LrgBagged Units':'Large Bags', 'X-LrgBagged Units':'XLarge Bags'}, inplace = True)
avocado_2020.rename(columns = {'Geography':'Region', 'Current Year Week Ending':'Date', 'Type':'Type', 'ASP Current Year':'AveragePrice',
       'Total Bulk and Bags Units':'Total Volume', '4046 Units':'4046', '4225 Units':'4225',
       '4770 Units':'4770', 'TotalBagged Units':'Total Bags', 'SmlBagged Units':'Small Bags',
       'LrgBagged Units':'Large Bags', 'X-LrgBagged Units':'XLarge Bags'}, inplace = True)<jupyter_output><empty_output><jupyter_text>" Year " And " Month " columns adding<jupyter_code>avocado_2015_to_2017['str_split'] = avocado_2015_to_2017.Date.str.split('-')
avocado_2015_to_2017['Month'] = avocado_2015_to_2017.str_split.str.get(1)
avocado_2015_to_2017.drop(['Date' , 'str_split'] , axis = 1 , inplace=True)
avocado_2015_to_2017.head()
avocado_2018['str_split'] = avocado_2018.Date.str.split(' ')
avocado_2018['Date'] = avocado_2018.str_split.str.get(0)
avocado_2018['str_split'] = avocado_2018.Date.str.split('-')
avocado_2018['Year'] = avocado_2018.str_split.str.get(0)
avocado_2018['Month'] = avocado_2018.str_split.str.get(1)
avocado_2018.drop(['Date' , 'str_split'] , axis = 1 , inplace=True)
avocado_2018.head()
avocado_2019['str_split'] = avocado_2019.Date.str.split(' ')
avocado_2019['Date'] = avocado_2019.str_split.str.get(0)
avocado_2019['str_split'] = avocado_2019.Date.str.split('-')
avocado_2019['Year'] = avocado_2019.str_split.str.get(0)
avocado_2019['Month'] = avocado_2019.str_split.str.get(1)
avocado_2019.drop(['Date' , 'str_split'] , axis = 1 , inplace=True)
avocado_2019.head()
avocado_2020['str_split'] = avocado_2020.Date.str.split(' ')
avocado_2020['Date'] = avocado_2020.str_split.str.get(0)
avocado_2020['str_split'] = avocado_2020.Date.str.split('-')
avocado_2020['Year'] = avocado_2020.str_split.str.get(0)
avocado_2020['Month'] = avocado_2020.str_split.str.get(1)
avocado_2020.drop(['Date' , 'str_split'] , axis = 1 , inplace=True)
avocado_2020.head()<jupyter_output><empty_output><jupyter_text>" Year " And " Month " columns value casting<jupyter_code>avocado_2015_to_2017['Month'] = avocado_2015_to_2017['Month'].astype('int64')
avocado_2015_to_2017.info()
avocado_2018['Year'] = avocado_2018['Year'].astype('int64')
avocado_2018['Month'] = avocado_2018['Month'].astype('int64')
avocado_2018.info()
avocado_2019['Year'] = avocado_2019['Year'].astype('int64')
avocado_2019['Month'] = avocado_2019['Month'].astype('int64')
avocado_2019.info()
avocado_2020['Year'] = avocado_2020['Year'].astype('int64')
avocado_2020['Month'] = avocado_2020['Month'].astype('int64')
avocado_2020.info()
avocado_2015_to_2017['Month'] = avocado_2015_to_2017['Month'].apply(lambda x: calendar.month_abbr[x])
avocado_2018['Month'] = avocado_2018['Month'].apply(lambda x: calendar.month_abbr[x])
avocado_2019['Month'] = avocado_2019['Month'].apply(lambda x: calendar.month_abbr[x])
avocado_2020['Month'] = avocado_2020['Month'].apply(lambda x: calendar.month_abbr[x])<jupyter_output><empty_output><jupyter_text>Fix " Type " column values<jupyter_code>avocado_2015_to_2017['Type'].value_counts()
avocado_2015_to_2017['Type'] = avocado_2015_to_2017.Type.str.title()
avocado_2015_to_2017['Type'].value_counts()
# 2018 is correct
avocado_2018['Type'].value_counts()
avocado_2019['Type'].value_counts()
avocado_2019['Type'] = avocado_2019.Type.str.replace(' ' , '')
avocado_2019['Type'].value_counts()
# 2020 is correct
avocado_2020['Type'].value_counts()<jupyter_output><empty_output><jupyter_text>Fix " Region " column value<jupyter_code>avocado_2015_to_2017['Region'].unique()
avocado_2018['Region'].unique()
avocado_2019['Region'].unique()
avocado_2020['Region'].unique()
avocado_2018['Region'] = avocado_2018.Region.str.replace(' ', '')
avocado_2018['Region'] = avocado_2018.Region.str.replace('/', '')
avocado_2018['Region'] = avocado_2018.Region.str.replace('.', '')
avocado_2019['Region'] = avocado_2019.Region.str.replace(' ', '')
avocado_2019['Region'] = avocado_2019.Region.str.replace('/', '')
avocado_2019['Region'] = avocado_2019.Region.str.replace('.', '')
avocado_2020['Region'] = avocado_2020.Region.str.replace(' ', '')
avocado_2020['Region'] = avocado_2020.Region.str.replace('/', '')
avocado_2020['Region'] = avocado_2020.Region.str.replace('.', '')
avocado_2015_to_2017['Region'].unique()
avocado_2018['Region'].unique()
avocado_2019['Region'].unique()
avocado_2020['Region'].unique()<jupyter_output><empty_output><jupyter_text>Concat 4 dataframe together<jupyter_code>frames = [avocado_2015_to_2017, avocado_2018, avocado_2019 , avocado_2020]
avocado_2015_to_2020 = pd.concat(frames , sort = False)
avocado_2015_to_2020.reset_index(inplace=True)
avocado_2015_to_2020.drop(['index'] , axis = 1, inplace=True)
avocado_2015_to_2020['Type'] = avocado_2015_to_2020.Type.str.replace(' ' , '')<jupyter_output><empty_output><jupyter_text>Convert dataframe to CSV<jupyter_code>avocado_2015_to_2020.to_csv('datasets/avocado_2015_to_2020.csv' , index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17792652552926524, 'fraction_numerical': 0.11908468244084683, 'mean_word_length': 8.930448222565687, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '408458', 'license': 'MIT', 'n_tokens_mistral': 3080, 'n_tokens_neox': 2327, 'n_words': 310, 'path': 'Avocada Data Cleaning .ipynb', 'repo_name': 'BahramJannesar/Avocado', 'size': '6424'}