<jupyter_start><jupyter_text>First Exploration Nov. 5 2020<jupyter_code>import csv
import numpy as np
import pandas as pd
# import geopandas as gpd
from IPython.display import Image
# from shapely.geometry import Point, Polygon
from math import factorial

import datetime
from datetime import date
import datetime
import time


import scipy
import os, os.path
import itertools
from statsmodels.sandbox.regression.predstd import wls_prediction_std
from sklearn.linear_model import LinearRegression
from patsy import cr

# from pprint import pprint
import matplotlib.pyplot as plt
import seaborn as sb


import sys


# search path for modules
# look @ https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
sys.path.append('/Users/hn/Documents/00_GitHub/Ag/Stop_the_Rot/')
import onion_core as oc
data_dir = "/Users/hn/Documents/01_research_data/Stop_the_Rot/files_from_team/Columbia_Basin_Farm_A_red_onion_data/"
file_names = ["ABC_Correct.xlsx", 
              "CLEANED (3) Real Time Soil Moisture data_Field A1.xlsx", 
              "CLEANED (3) Real Time Soil Moisture data_Field A2.xlsx", 
              "CLEANED (3) Real Time Soil Moisture data_Field A3.xlsx", 
              "CLEANED (3) Real Time Soil Moisture data_Field A4.xlsx",
              "CLEANED (3) Real Time Soil Moisture data_Field A5.xlsx",]


A1 = oc.read_xlsx_with_only_1sheet(path_n_fileName = data_dir + file_names[1], head_count=0)
A2 = oc.read_xlsx_with_only_1sheet(path_n_fileName = data_dir + file_names[2], head_count=0)
A3 = oc.read_xlsx_with_only_1sheet(path_n_fileName = data_dir + file_names[3], head_count=0)
A4 = oc.read_xlsx_with_only_1sheet(path_n_fileName = data_dir + file_names[4], head_count=0)
A5 = oc.read_xlsx_with_only_1sheet(path_n_fileName = data_dir + file_names[5], head_count=0)
A1.head(3)
A1.tail(3)
# print (min(A1.Date))
# print (max(A1.Date))
# print ("------------------------")
# print (min(A2.Date))
# print (max(A2.Date))
# print ("------------------------")
# print (min(A3.Date))
# print (max(A3.Date))
# print ("------------------------")
# print (min(A4.Date))
# print (max(A4.Date))
# print ("------------------------")
# print (min(A5.Date))
# print (max(A5.Date))
# v = [1, 2 ,3]
# print (min(A1.Date))
# print (min(A2.Date))
# print (min(A3.Date))
# print (min(A4.Date))
# print (min(A5.Date))

# print ("------------------------")

# print (max(A1.Date))
# print (max(A2.Date))
# print (max(A3.Date))
# print (max(A4.Date))
# print (max(A5.Date))
A1.Date[3] - A1.Date[2]
# A1["Date_col"] = A1.Date
# A1['Date'] = A1['Date'].dt.date # converts timeStamp to datetime and tosses hour minute.
# fig, ax1 = plt.subplots(1,1,figsize=(16,4));
# A1.plot(x='Date', y = 'Sensor1', legend = "Sensor1", ax=ax1);
# A1.plot(x='Date', y = 'Sensor2', legend = "Sensor2", ax=ax1);
# A1.plot(x='Date', y = 'Sensor3', legend = "Sensor3", ax=ax1);
# A1.plot(x='Date', y = 'Sensor4', legend = "Sensor4", ax=ax1);
# ax1.set_title("A1 data");
# fig, ax1 = plt.subplots(1,1,figsize=(16,4));
# A2.plot(x='Date', y = 'Sensor1', legend = "Sensor1", ax=ax1);
# A2.plot(x='Date', y = 'Sensor2', legend = "Sensor2", ax=ax1);
# A2.plot(x='Date', y = 'Sensor3', legend = "Sensor3", ax=ax1);
# A2.plot(x='Date', y = 'Sensor4', legend = "Sensor4", ax=ax1);
# ax1.set_title("A2 data");
# fig, ax1 = plt.subplots(1,1,figsize=(16,4));
# A3.plot(x='Date', y = 'Sensor1', legend = "Sensor1", ax=ax1);
# A3.plot(x='Date', y = 'Sensor2', legend = "Sensor2", ax=ax1);
# A3.plot(x='Date', y = 'Sensor3', legend = "Sensor3", ax=ax1);
# A3.plot(x='Date', y = 'Sensor4', legend = "Sensor4", ax=ax1);
# ax1.set_title("A3 data");
fig, axs = plt.subplots(3, 2, figsize=(20,12),
                        sharex='col', sharey='row',
                        gridspec_kw={'hspace': 0.1, 'wspace': .1});

(ax1, ax2), (ax3, ax4), (ax5, ax6) = axs;
ax1.grid(True); ax2.grid(True); ax3.grid(True); ax4.grid(True); ax5.grid(True); ax6.grid(True);

ax1.plot(A1.Date.values, A1.Sensor1.values, label="Sensor 1")
ax1.plot(A1.Date.values, A1.Sensor2.values, label="Sensor 2")
ax1.plot(A1.Date.values, A1.Sensor3.values, label="Sensor 3")
ax1.plot(A1.Date.values, A1.Sensor4.values, label="Sensor 4")
ax1.legend(loc="best");
ax1.set_title("A1");
ax1.ylim = (0, 1.2)
ax1.set_ylim([0.2, 1.3])

ax2.plot(A2.Date.values, A2.Sensor1.values, label="Sensor 1")
ax2.plot(A2.Date.values, A2.Sensor2.values, label="Sensor 2")
ax2.plot(A2.Date.values, A2.Sensor3.values, label="Sensor 3")
ax2.plot(A2.Date.values, A2.Sensor4.values, label="Sensor 4")
ax2.legend(loc="best");
ax2.set_title("A2");
ax2.set_ylim([0.2, 1.3])

ax3.plot(A3.Date.values, A3.Sensor1.values, label="Sensor 1")
ax3.plot(A3.Date.values, A3.Sensor2.values, label="Sensor 2")
ax3.plot(A3.Date.values, A3.Sensor3.values, label="Sensor 3")
ax3.plot(A3.Date.values, A3.Sensor4.values, label="Sensor 4")
ax3.legend(loc="best");
ax3.set_title("A3");
ax3.set_ylim([0.2, 1.3])

ax4.plot(A4.Date.values, A4.Sensor1.values, label="Sensor 1")
ax4.plot(A4.Date.values, A4.Sensor2.values, label="Sensor 2")
ax4.plot(A4.Date.values, A4.Sensor3.values, label="Sensor 3")
ax4.plot(A4.Date.values, A4.Sensor4.values, label="Sensor 4")
ax4.legend(loc="best");
ax4.set_title("A4");
ax4.set_ylim([0.2, 1.3])

ax5.plot(A5.Date.values, A5.Sensor1.values, label = "Sensor 1")
ax5.plot(A5.Date.values, A5.Sensor2.values, label = "Sensor 2")
ax5.plot(A5.Date.values, A5.Sensor3.values, label = "Sensor 3")
ax5.plot(A5.Date.values, A5.Sensor4.values, label = "Sensor 4")
ax5.legend(loc="best");
ax5.set_title("A5");
ax5.set_ylim([0.2, 1.3])


# A3.plot(x='Date', y = 'Sensor1', legend = "Sensor1", ax=ax1);

# axs[0].bar(names, values)
# axs[1].scatter(names, values)
# axs[2].plot(names, values)
# fig.suptitle('Title here');

file_name = data_dir + "sensor_readings.png"
plt.savefig(fname = file_name, dpi=600, bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>CLEANED (3) Station ABC Weather Data 2017 file<jupyter_code>file_names
# file_names[0]
ABC_2016_2017 = oc.read_ABC_weather_XLSfile(an_add_fileName = data_dir + 
                                            "CLEANED (3) Station ABC Weather Data 2017.xlsx")
ABC_2016_2017.head(2)
harvest_sample_2017_name = "CLEANED(2) 2017 Red Onion Harvest Samples.xlsx"
A = pd.read_excel(io=data_dir + harvest_sample_2017_name, header = 0)
harvest_sample_2017 = oc.read_xlsx_with_only_1sheet(path_n_fileName= data_dir + harvest_sample_2017_name, 
                                                    head_count = 0)
harvest_sample_2017.sort_values(by = ['Field code', 'Date'], inplace=True)
harvest_sample_2017.columns
harvest_sample_2017.head(2)<jupyter_output><empty_output><jupyter_text>Plan for this cell on:   **Make the monthly data points:**   - Read Cleaned 3 Farm A irrigation Applied 2017   - Read Station ABC   - Create a dataframe using them   - Read the file with decay in it at harvest time.   - Apply PCA to dataframe and pick the top 1 or 2 components to explore.    Do we need to produce something similar to accumulated GDD for solar or any other variable?<jupyter_code>python_data_dir = data_dir + "Clean_For_Python/"
python_data_dir
clean_dir = "/Users/hn/Documents/01_research_data/Stop_the_Rot/Clean_For_Python/"<jupyter_output><empty_output><jupyter_text>ABC_weather  - ABC weather includes variables `[Max Temp, Max Temp, Avg Temp, Dew Point, Rain, Wind, Solar, ETp, GDD]`.    Use them to creafe monthly data: 1. `Monthly average temp` 2. `monthly average dew`  3. `cumulative rain` 4. `cumulative GDD`<jupyter_code>ABC_2016_2017 = oc.read_ABC_weather_XLSfile(an_add_fileName = clean_dir + "ABC_Correct.xlsx",
                                            header_rows = 1, 
                                            skip_Rows = 3)<jupyter_output><empty_output><jupyter_text>We have to do this next step since we updated the damn Pandas<jupyter_code>ABC_2016_2017['Date'] = pd.to_datetime(ABC_2016_2017.Date, format='%Y-%m-%d %H:%M:%S')
ABC_2016_2017['Date'] = ABC_2016_2017['Date'].dt.strftime('%Y-%m-%d')
ABC_2016_2017['Date'] = pd.to_datetime(ABC_2016_2017.Date, format='%Y-%m-%d')
ABC_2016_2017['month'] = ABC_2016_2017['Date'].dt.month
ABC_2016_2017['year'] = ABC_2016_2017['Date'].dt.year
ABC_2016_2017.head(15)
ABC_2016_2017.Date[1].year
#
# convert all columns to numeric, so we can use groupby operations.
#
oc.convert_to_numerictype(ABC_2016_2017, ABC_2016_2017.columns[1:9])

ABC_2016_2017.head(2)
# ABC_2016_2017.drop(['Date'], axis=1, inplace=True)
ABC_2016_2017.GDD = ABC_2016_2017.GDD.astype(float)
# for ii in np.arange(len(ABC_2016_2017.GDD)):
#     if type(ABC_2016_2017.GDD[ii]) != float:
#         print (ii)
#         print (ABC_2016_2017.iloc[ii])
#         print (type(ABC_2016_2017.GDD[ii]) )

mean_dataF = oc.group_by_compute_stats(a_dataF = ABC_2016_2017, 
                                       stat = 'mean', 
                                       stat_columns = ['Average Temp (F)', 'Dew Point', 'Wind (Miles)'], 
                                       group_by_cols = ['year', 'month'])
cumSum_dataF = oc.group_by_compute_stats(a_dataF = ABC_2016_2017, 
                                         stat = 'sum', 
                                         stat_columns = ['GDD', 'Rain (Inch)'], 
                                         group_by_cols = ['year', 'month'])
processed_table = pd.merge(left = cumSum_dataF, right = mean_dataF, how='left', on = ['year', 'month'])
processed_table<jupyter_output><empty_output><jupyter_text>Cleaned 3 Farm A irrigation Applied 2017<jupyter_code>dd = "/Users/hn/Documents/01_research_data/Stop_the_Rot/Clean_For_Python/"
irrigation_fileN = "CLEANED (3) Farm A Irrigation Applied 2017 selected fields_Python.xlsx"
xl = pd.ExcelFile(dd + irrigation_fileN)
EX_sheet_names = xl.sheet_names
EX_sheet_names
Irrigation_2017 = pd.read_excel(io = dd + irrigation_fileN, 
                                header = 0, 
                                sheet_name = "(clean) Year Summary", 
                                skiprows = 0)
Irrigation_2017.shape
ii=2
print (Irrigation_2017.Date[ii])
print (pd.to_datetime(Irrigation_2017.Date[ii]).month)
print (pd.to_datetime(Irrigation_2017.Date[ii]).day)

Irrigation_2017.Date[20]
print (pd.to_datetime(Irrigation_2017.Date[20]).month)
Irrigation_2017.head(2)<jupyter_output><empty_output><jupyter_text>replace NAs with zeros<jupyter_code>Irrigation_2017_NA_0 = Irrigation_2017.copy()
Irrigation_2017_NA_0.sort_values(by = 'Date', inplace=True)
Irrigation_2017_NA_0['irrig_inch'] = Irrigation_2017_NA_0['irrig_inch'].fillna(0)

Irrigation_2017_NA_0['irr_cum_sum'] = Irrigation_2017_NA_0.groupby(['Field'])['irrig_inch'].cumsum()
# Irrigation_2017[Irrigation_2017['irrig_inch'].isnull()]
Irrigation_2017_NA_0.head(3)
df = Irrigation_2017_NA_0.copy();

fig, ax = plt.subplots(figsize=(15,7))

df.set_index('Date', inplace=True);
df.groupby('Field')['irr_cum_sum'].plot(legend=True);

ax.grid(True);
ax.set_title("Cumulative irrigation");


file_name = python_data_dir + "Cumulative_irrigation.png"
# plt.savefig(fname = file_name, dpi=600, bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>- **A1**: Apr 1 - Sep 6   - **A2**: Mar 25 - Sep 6   - **A3**: Mar 15 - Sep 13   - **A4**: Mar 21 - Sep 15   - **A5**: Mar 16 - Sep 23<jupyter_code>planting_dates = [datetime.date(2017, 4, 1),
                  datetime.date(2017, 3, 25),
                  datetime.date(2017, 3, 15),
                  datetime.date(2017, 3, 21),
                  datetime.date(2017, 3, 16)]

harvesting_dates = [datetime.date(2017, 9, 6),
                    datetime.date(2017, 9, 6),
                    datetime.date(2017, 9, 13),
                    datetime.date(2017, 9, 15),
                    datetime.date(2017, 9, 23)]

field_names = ['A1', 'A2', 'A3', 'A4', 'A5']

data1 = {'field_names': field_names, 'planting dates': planting_dates, 'harvesting dates': harvesting_dates}

Planting_and_Harvesting_dates = pd.DataFrame(data1)
planting_dates = [datetime.date(2017, 4, 1),
                  datetime.date(2017, 3, 25),
                  datetime.date(2017, 3, 15),
                  datetime.date(2017, 3, 21),
                  datetime.date(2017, 3, 16)]

harvesting_dates = [datetime.date(2017, 9, 6),
                    datetime.date(2017, 9, 6),
                    datetime.date(2017, 9, 13),
                    datetime.date(2017, 9, 15),
                    datetime.date(2017, 9, 23)]

field_names = ['A1', 'A2', 'A3', 'A4', 'A5']

planting_table = {'Field': field_names, 'planting dates': planting_dates, 'harvesting dates': harvesting_dates}
planting_table = {'Field': field_names, 'planting dates': planting_dates, 'harvesting dates': harvesting_dates}

Planting_and_Harvesting_dates = pd.DataFrame(data1)
# df = Irrigation_2017_NA_0.copy();

# fig, ax = plt.subplots(figsize=(15,7))

# df.set_index('Date', inplace=True);
# df.groupby('Field')['irr_cum_sum'].plot(legend=True);

# # add planting dots
# plt.scatter(x = Planting_and_Harvesting_dates.loc[0, 'planting dates'], y = 5.7, linewidths=3)
# plt.scatter(x = Planting_and_Harvesting_dates.loc[1, 'planting dates'], y = 2, linewidths=3)
# plt.scatter(x = Planting_and_Harvesting_dates.loc[2, 'planting dates'], y = 1.2, linewidths=3)
# plt.scatter(x = Planting_and_Harvesting_dates.loc[3, 'planting dates'], y = 0, linewidths=3)
# plt.scatter(x = Planting_and_Harvesting_dates.loc[4, 'planting dates'], y = 0, linewidths=3)

# plt.scatter(x = Planting_and_Harvesting_dates.loc[0, 'harvesting dates'], 
#             y = 29.5, 
#             linewidths=3, 
#             c = 'cornflowerblue')

# plt.scatter(x = Planting_and_Harvesting_dates.loc[1, 'harvesting dates'], 
#             y = 24.2, 
#             linewidths=3, 
#             c = 'orange')

# plt.scatter(x = Planting_and_Harvesting_dates.loc[2, 'harvesting dates'], 
#             y = 25, 
#             linewidths=3, 
#             c = 'green')

# plt.scatter(x = Planting_and_Harvesting_dates.loc[3, 'harvesting dates'], 
#             y = 17, 
#             linewidths=3, 
#             c = 'red')

# plt.scatter(x = Planting_and_Harvesting_dates.loc[4, 'harvesting dates'], 
#             y = 17, 
#             linewidths=3, 
#             c = 'purple')

# ax.grid(True);
# ax.set_title("Cumulative irrigation (dots are planting and harvesting dates)");


# file_name = python_data_dir + "Cumulative_irrigation_wPantingHarvesting.png"
# plt.savefig(fname = file_name, dpi=600, bbox_inches='tight')
# plt.show()
planting_dates = [datetime.date(2017, 4, 1),
                  datetime.date(2017, 3, 25),
                  datetime.date(2017, 3, 15),
                  datetime.date(2017, 3, 21),
                  datetime.date(2017, 3, 16)]

planting_y = [5.7, 2, 1.2, 0, 0]

harvesting_dates = [datetime.date(2017, 9, 6),
                    datetime.date(2017, 9, 6),
                    datetime.date(2017, 9, 13),
                    datetime.date(2017, 9, 15),
                    datetime.date(2017, 9, 23)]

harvesting_y = [29.5, 24.2, 25, 17, 17]

field_names = ['A1', 'A2', 'A3', 'A4', 'A5']

colors = ['cornflowerblue', 'orange', 'green', 'red', 'purple']

planting_table = {'Field': field_names, 'planting dates': planting_dates, 'y': planting_y, 'colors':colors}
harvesting_table = {'Field': field_names, 'harvesting dates': harvesting_dates, 'y': harvesting_y, 'colors':colors}

planting_table = pd.DataFrame(planting_table)
harvesting_table = pd.DataFrame(harvesting_table)

planting_table.sort_values(by = 'planting dates', inplace=True)
harvesting_table.sort_values(by = 'harvesting dates', inplace=True)
import matplotlib.pylab as pylab
params = {'legend.fontsize': 'x-large',
          #'figure.figsize': (15, 5),
          'axes.labelsize': 24, # changes date (a), (b), (c), (d) on x-axis, EVI, NDVI on y axis. 
                               # The smaller this number the bigger the labels!!
          'axes.titlesize': 24, # changes titles, bean dry for example. 
                               # This line and the one above are connected, problematic
          'xtick.labelsize':15,
          'ytick.labelsize':15
         }


df = Irrigation_2017_NA_0.copy();

fig, ax = plt.subplots(figsize=(15,7))

df.set_index('Date', inplace=True);
df.groupby('Field')['irr_cum_sum'].plot(legend=True);

# add planting dots
plt.scatter(x = planting_table['planting dates'], 
            y = planting_table['y'], 
            linewidths=3, 
            c = planting_table.colors)

plt.scatter(x = harvesting_table['harvesting dates'], 
            y = harvesting_table['y'], 
            linewidths=3,
            c = harvesting_table.colors)

ax.set(xlabel = 'date', ylabel = 'inch')
ax.xaxis.labelpad = 15
ax.yaxis.labelpad = 15


ax.grid(True);
ax.set_title("Cumulative irrigation (dots are planting and harvesting dates)");

file_name = dd + "Cumulative_irrigation_wPantingHarvesting_2.png"
# plt.savefig(fname = file_name, dpi=600, bbox_inches='tight')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16016499705362405, 'fraction_numerical': 0.06269888037713613, 'mean_word_length': 3.5171679531541122, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '18234', 'license': 'MIT', 'n_tokens_mistral': 7036, 'n_tokens_neox': 6337, 'n_words': 1221, 'path': 'Stop_the_Rot/00_First_Exploration/01_Exploration.ipynb', 'repo_name': 'HNoorazar/Kirti', 'size': '16970'}