<jupyter_start><jupyter_text>upload csv file<jupyter_code>file_path = fps.check_file("Sludge age and f_m.csv")
df = pd.read_csv(file_path)
df.set_index("date", inplace=True)
df.index = pd.to_datetime(df.index, dayfirst=True)
df<jupyter_output><empty_output><jupyter_text>Remove out of range values<jupyter_code>def check_values_range(df: pd.DataFrame) -> pd.DataFrame:
    """
    Check and replace incorrect values with nan.
    (Zero values or values out of defined ranges)
    """
    df.replace(0, np.nan, inplace=True)
    ranges_lst = [(0.05, 0.6), (0.5, 9)]
    col_names_lst = ["F_M", "Sludge Age"]
    
    for i in range(2):
        name = col_names_lst[i]
        low = ranges_lst[i][0]
        high = ranges_lst[i][1]
        df.loc[:, df.columns.str.contains(name)] = df.loc[
            :, df.columns.str.contains(name)
        ].apply(lambda x: [value if low < value < high else np.nan for value in x])

    return df
df = check_values_range(df)<jupyter_output><empty_output><jupyter_text>Interpolate missing data by methode time<jupyter_code>df.interpolate(inplace=True, method="time")
df = df.fillna(method='bfill')<jupyter_output><empty_output><jupyter_text>Visual data<jupyter_code>df
fig, ax = plt.subplots()
fig.set_size_inches(18.5, 10.5)
_=sns.scatterplot(data=df, x=df.index, y="Sludge Age - A", ax=ax)
_=sns.scatterplot(data=df, x=df.index, y="Sludge Age - B", ax=ax)
plt.ylabel("Sludge age")
plt.legend(["Sludge Age - A", "Sludge Age - B"])
plt.xlim(df.index.min(), df.index.max())
_=plt.ylim(0, 10)
fig, ax = plt.subplots()
fig.set_size_inches(18.5, 10.5)
_=sns.scatterplot(data=df, x=df.index, y="F_M - A", ax=ax)
_=sns.scatterplot(data=df, x=df.index, y="F_M - B", ax=ax)
plt.ylabel("F/M")
plt.legend(["F_M - A", "F_M - B"])
plt.xlim(df.index.min(), df.index.max())
_=plt.ylim(0, 0.6)<jupyter_output><empty_output><jupyter_text>Remove local outliers<jupyter_code>def IQR_cal (df: pd.DataFrame):
    Q1 = df.quantile(0.25)
    Q3 = df.quantile(0.75)
    IQR = Q3 - Q1
    return IQR, Q1, Q3
for i in range(len(df)):
    window = 10
    jump = 5
    j=i*jump
    if j+window<len(df):
        IQR, Q1, Q3 = IQR_cal(df.iloc[j:j+window,:])
        k=j
        while k<j+window:
            for z in range(4):
                if (df.iloc[k,z] < (Q1[z] - 1 * IQR[z]) or (df.iloc[k,z] > (Q3[z] + 1 * IQR[z]))):
                    df.iloc[k,z] = np.nan
            k=k+1<jupyter_output><empty_output><jupyter_text>interpolate and fill values insted of nan<jupyter_code>df.interpolate(inplace=True, method="time")
df = df.fillna(method='bfill')<jupyter_output><empty_output><jupyter_text>Revisual data<jupyter_code>fig, ax = plt.subplots()
fig.set_size_inches(18.5, 10.5)
_=sns.scatterplot(data=df, x=df.index, y="Sludge Age - A", ax=ax)
_=sns.scatterplot(data=df, x=df.index, y="Sludge Age - B", ax=ax)
plt.ylabel("Sludge age")
plt.legend(["Sludge Age - A", "Sludge Age - B"])
plt.xlim(df.index.min(), df.index.max())
_=plt.ylim(0, 10)
fig, ax = plt.subplots()
fig.set_size_inches(18.5, 10.5)
_=sns.scatterplot(data=df, x=df.index, y="F_M - A", ax=ax)
_=sns.scatterplot(data=df, x=df.index, y="F_M - B", ax=ax)
plt.ylabel("F/M")
plt.legend(["F_M - A", "F_M - B"])
plt.xlim(df.index.min(), df.index.max())
_=plt.ylim(0, 0.6)
df = df.round(decimals=2)
df
df.index = pd.to_datetime(df.index, dayfirst=True).date
df
df.index.name = 'date'
df.to_csv(pathlib.Path("clean_tables/sludge_age_f_m.csv"), index=True)
# fig, ax1 = plt.subplots()
# fig.set_size_inches(18.5, 10.5)
# _=sns.scatterplot(data=df, x=df.index, y="Sludge Age - A", ax=ax1)
# ax1.set_ylabel("Sludge age")
# ax1.legend(["Sludge age - A"])
# ax2 = ax1.twinx()
# _=sns.scatterplot(data=df, x=df.index, y="F_M - A", ax=ax2, color = 'tab:red')
# ax2.set_ylabel("F_M - A")
# ax2.legend(["F/M - A"])
# plt.xlim(df.index.min(), df.index.max())
df['Month'] = pd.DatetimeIndex(df.index).month
df['Year'] = pd.DatetimeIndex(df.index).year
df['F_M'] = df.iloc[:, np.r_[0: 2]].mean(axis=1)
df['Sludge_age'] = df.iloc[:, np.r_[2: 4]].mean(axis=1)
df
table = pd.pivot_table(df, values=df.columns[6], index=['Month'], columns=['Year'])
ax = sns.heatmap(table, cmap="rocket_r")
plt.title(df.columns[6])
table = pd.pivot_table(df, values=df.columns[7], index=['Month'], columns=['Year'])
ax = sns.heatmap(table, cmap="rocket_r")
plt.title(df.columns[7])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19774660841572775, 'fraction_numerical': 0.022303977925959992, 'mean_word_length': 4.423940149625935, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '516834', 'license': 'MIT', 'n_tokens_mistral': 1990, 'n_tokens_neox': 1915, 'n_words': 328, 'path': 'sludge age and f_m upload.ipynb', 'repo_name': 'efratkohen/Project', 'size': '4349'}