<jupyter_start><jupyter_text>Import required library<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Import the required datas<jupyter_code>#import zipfile
#zip_file = zipfile.ZipFile('retail-data-analytics.zip', 'r')
#zip_file.namelist()
# our datas are in datas folder in same directory so 
df = pd.read_csv("datas/Features_data_set.csv")
sales =  pd.read_csv("datas/sales_data_set.csv")
stores = pd.read_csv("datas/stores_data_set.csv")
#check the types of Column format
df.info()      # We can see Date is in object format
sales.info()  # We can see Date is in object format
stores.info()   # We can see Date is in object format<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 8190 entries, 0 to 8189
Data columns (total 12 columns):
Store           8190 non-null int64
Date            8190 non-null object
Temperature     8190 non-null float64
Fuel_Price      8190 non-null float64
MarkDown1       4032 non-null float64
MarkDown2       2921 non-null float64
MarkDown3       3613 non-null float64
MarkDown4       3464 non-null float64
MarkDown5       4050 non-null float64
CPI             7605 non-null float64
Unemployment    7605 non-null float64
IsHoliday       8190 non-null bool
dtypes: bool(1), float64(9), int64(1), object(1)
memory usage: 712.0+ KB
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 421570 entries, 0 to 421569
Data columns (total 5 columns):
Store           421570 non-null int64
Dept            421570 non-null int64
Date            421570 non-null object
Weekly_Sales    421570 non-null float64
IsHoliday       421570 non-null bool
dtypes: bool(1), float64(1), int64(2), object(1)
memory usage: 13.3+[...]<jupyter_text>Change the date column format into match date format<jupyter_code>df["Date"] = pd.to_datetime(df["Date"])
sales["Date"]  =  pd.to_datetime(sales["Date"])
#stores["Date"] =  pd.to_datetime(stores["Date"])
df.Date.head()  #We can see the dtype is datetime64[ns]<jupyter_output><empty_output><jupyter_text>Check the datas shape<jupyter_code>print(df.shape, sales.shape, stores.shape)
print(sales[0:4].Date,'\n\n\n' ,sales[-3:].Date)<jupyter_output>0   2010-05-02
1   2010-12-02
2   2010-02-19
3   2010-02-26
Name: Date, dtype: datetime64[ns] 


 421567   2012-12-10
421568   2012-10-19
421569   2012-10-26
Name: Date, dtype: datetime64[ns]<jupyter_text>Merge the data into Unique DataFrame<jupyter_code>sales.columns,"*-"*50 , df.columns
# We are merging same column head, we know the data sizes are different in various columns<jupyter_output><empty_output><jupyter_text>while runnning the below code you must be care full and run only once beceause the datas will be over copied<jupyter_code>df_ = pd.merge(sales,df,on = ["Store","Date", "IsHoliday"], how="left")
df_ = pd.merge(df_, stores, on= ["Store"],how = "left")
df = df_ #df is no more use

df.shape
# the above will only added some columns only 
df.isna().sum()<jupyter_output><empty_output><jupyter_text>We can see that MarkDown1, 2 , 3, 4, 5 columns have na value to much , so we have to manage them separately<jupyter_code>df= df.fillna(0)
df["Temperature"] = (df["Temperature"]-32)*5./9. # fer cen
types_encoded , types = df["Type"].factorize()
df["Type"] = types_encoded # #types_encoded is numerical values and types is original
#df.head()
"""
labels, uniques = pd.factorize(['b', 'b', 'a', 'c', 'b'])
>>> labels
array([0, 0, 1, 2, 0])
>>> uniques
array(['b', 'a', 'c'], dtype=object)
"""

df.isna().sum() # We can see there is no na values
df.head()
df["Type"].unique()# There are only 3 types<jupyter_output><empty_output><jupyter_text>Check data and features<jupyter_code>print("training_data duplicated:{}".format(df.duplicated().sum()))
df.shape
df.drop_duplicates(inplace = True)
df.shape
df.describe()
tab_info = pd.DataFrame(df.dtypes).T.rename(index = {0:'column Type'})
tab_info = tab_info.append(pd.DataFrame(df.isnull().sum()).T.rename(index = {0:"null values(nb)"}))

tab_info = tab_info.append(pd.DataFrame(df.isnull().sum()/df.shape[0]*100).T.rename(index = {0: "null values (%)"}))
tab_info<jupyter_output><empty_output><jupyter_text>Gain Some Graphical insight **Plot** some of the variable present in the data such as ```temperature , fuel price , CPI , unemployment rate ```<jupyter_code>df[["Date","Temperature", "Fuel_Price","CPI", "Unemployment" , 'MarkDown1', "MarkDown2" , "MarkDown3" , "MarkDown4" , "MarkDown5"]].plot(x = "Date",subplots = True, figsize = [10,8])
plt.show()<jupyter_output><empty_output><jupyter_text>**Sum the weekly sales from every shop to get the total weekly sales volume  of the retail chain**<jupyter_code>df_average_sales_week = df.groupby(by = ['Date'],as_index=False)["Weekly_Sales"].sum()
df_average_sales = df_average_sales_week.sort_values("Weekly_Sales", ascending = False)

plt.figure(facecolor="gold")
plt.figure(figsize = (210,25))
plt.plot(df_average_sales_week.Date, df_average_sales_week.Weekly_Sales,"r")

plt.show()<jupyter_output><empty_output><jupyter_text>Get More insights<jupyter_code># The more remunerative weeks
df_average_sales.head()
#df_average_sales.max()
df_average_sales[::3].size   # With the gap of 2
df_average_sales.shape
# The least remunerative weeks
df_average_sales[::-1].head()
ts = df_average_sales_week.set_index("Date")
#ts = ts.resample("H").ffill()
#ts = ts.resample("W").sum()
ts.head()
# The least remunerative weeks
df_average_sales [::-1].head()
# Top performing type of stores in term of sales
df_top_stores = df.groupby(by=["Type"],as_index = False)["Weekly_Sales"].sum()
df_top_stores.sort_values("Weekly_Sales", ascending = False)[:3]
# Top performing stores in term of sales
df_top_stores = df.groupby(by=["Store"], as_index = False)["Weekly_Sales"].sum()
df_top_stores.sort_values("Weekly_Sales",ascending = False)[:3]<jupyter_output><empty_output><jupyter_text>Forecast of the total sales volume<jupyter_code>from statsmodels.graphiics.tsaplots import acf, pacf , plot_acf , plot_pacf
fig, axes = plt.subplots(1,2,figsize= (20,5))
plot_acf(ts, lags = 100 , ax = axes[0])
plot_pacf(ts,lags=100, ax=axes[1])
plt.show()<jupyter_output><empty_output><jupyter_text>Model definition<jupyter_code>from sklearn.linear_model import LinearRegression

def fit_ar_model(ts,orders):
    x= np.array([ts.values[(i-orders)].squeeze() if i>=np.max(orders) else np.array(len(orders)*[np.nan]) for i in range(len(ts))])
    mask = ~np.isnan(x[:,:1]).squeeze()
    y = ts.values
    lin_reg = LinearRegression()
    lin_reg.fit(x[mask],y[mask])
    print(lin_reg.coef_ , lin_reg.intercept_)
    print("Score factor :%.2f"%lin_reg.score(x[mask],y[mask]))
    
    return lin_reg.coef_, lin_reg.intercept_

def predict_ar_model(ts, orders, coef, intercept):
    return np.array([np.sum(np.dot(coef, ts.values[(i-orders)].squeeze())) + intercept  if i >= np.max(orders) else np.nan for i in range(len(ts))])<jupyter_output><empty_output><jupyter_text>Given Seasonality observed from the ACF and the PACF function , the AR model is implemented including seasonality from weeks(1,6,52)<jupyter_code>import numpy as np
orders = np.array([1,6,52])
coef , intercept = fit_ar_model(ts,orders)
pred = pd.DataFrame(index = ts.index , data = predict_ar_model(ts,orders, coef , intercept))
plt.figure(figsize=(20,5))
plt.plot(ts,"o")
plt.plot(pred)
plt.show()
diff  = (ts["Weekly_Sales"]-pred[0])/ts["Weekly_Sales"]
print("AR Residuals : avg%.2f, std %.2f" %(diff.mean(), diff.std()))
plt.figure(figsize = (20,5))
plt.plot(diff, c = "orange")
plt.grid()
plt.show()<jupyter_output>AR Residuals : avg-0.00, std 0.07<jupyter_text>Forecast the store-wise sales volume<jupyter_code>df20 = df.where(df["Store"]==20)
df20 = df20.dropna()
df20 = df20.groupby(by = ["Date"], as_index = False)["Weekly_Sales"].sum()
df20 = df20.set_index("Date")
df20.head()
plt.figure(figsize = (20,5))
plt.plot(df20.index , df20.values)
plt.show()
fig, axes = plt.subplots(1,2,figsize = (20,5))
plot_acf(df20.values, lags = 100, alpha = 0.05, ax = axes[0])   #from stat model
plot_pacf(df20.values, lags = 100 , alpha = 0.5 , ax = axes[1])  #From stat model
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1496014714898835, 'fraction_numerical': 0.0457388105456775, 'mean_word_length': 5.598705501618123, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '155496', 'license': 'MIT', 'n_tokens_mistral': 3280, 'n_tokens_neox': 2997, 'n_words': 693, 'path': 'Sales_report_Korea.ipynb', 'repo_name': 'krishdb38/sales_data_pandas', 'size': '8155'}