<jupyter_start><jupyter_text>This is scratch to do maximum in 10-days intervals.<jupyter_code>import csv
import numpy as np
import pandas as pd
# import geopandas as gpd
from IPython.display import Image
# from shapely.geometry import Point, Polygon
from math import factorial
import scipy
import scipy.signal
import os, os.path

from datetime import date
import datetime
import time

from statsmodels.sandbox.regression.predstd import wls_prediction_std
from sklearn.linear_model import LinearRegression
from patsy import cr

# from pprint import pprint
import matplotlib.pyplot as plt
import seaborn as sb

import sys

sys.path.append('/Users/hn/Documents/00_GitHub/Ag/remote_sensing/python/')
import remote_sensing_core as rc
import remote_sensing_core as rcp

start_time = time.time()
data_dir = "/Users/hn/Documents/01_research_data/" + \
           "remote_sensing/01_NDVI_TS/00_Eastern_WA_withYear/2Years/"

param_dir = "/Users/hn/Documents/00_GitHub/Ag/remote_sensing/parameters/"<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>####################################################################################
###
###      Parameters                   
###
####################################################################################

irrigated_only = 0
SF_year = 2017
indeks = "EVI"

regular_window_size = 10<jupyter_output><empty_output><jupyter_text>Read the data<jupyter_code>f_name = "Eastern_WA_" + str(SF_year) + "_70cloud_selectors.csv"
a_df = pd.read_csv(data_dir + f_name, low_memory=False)

##################################################################
##################################################################
####
####  plots has to be exact. So, we need 
####  to filter out NASS, and filter by last survey date
####
##################################################################
##################################################################

a_df = a_df[a_df['county']== "Grant"] # Filter Grant
# a_df = rc.filter_out_NASS(a_df) # Toss NASS
# a_df = rc.filter_by_lastSurvey(a_df, year = SF_year) # filter by last survey date
a_df['SF_year'] = SF_year<jupyter_output><empty_output><jupyter_text>Functions  Get a field's data<jupyter_code>a_df.reset_index(drop=True, inplace=True)
a_df_1 = a_df[a_df.ID == a_df.ID[0]]
a_df_1.shape
a_df_1 = rc.initial_clean_EVI(a_df_1)
# a_df_1.sort_values(by=['system_start_time'], inplace=True)
a_df_1.sort_values(by=['image_year', 'doy'], inplace=True)

a_df_1 = rc.correct_timeColumns_dataTypes(a_df_1)

a_df_1.reset_index(drop=True, inplace=True)
print(a_df_1.shape)
# a_df_1.head(2)
a_df_1.system_start_time[0]
A = rc.regularize_movingWindow_windowSteps_2Yrs(one_field_df = a_df_1, SF_yr=SF_year, idks=indeks, window_size=10)
a_df_1.image_year.unique()
A.shape
print (a_field_df.shape)
print (regular_df.shape)
outName = "/Users/hn/Documents/01_research_data/remote_sensing/test_data/a_regularized_TS.csv"
regularized_TS.to_csv(outName, index=False)<jupyter_output><empty_output><jupyter_text>Create Aeolus Enviornment, and see things works<jupyter_code>first_10_IDs = a_df.ID.unique()[:10]
an_EE_TS = a_df[a_df.ID.isin(first_10_IDs) ]
indeks = "EVI"
county = "Grant"
SF_year = 2017
regular_window_size = 10
########################################################################################

an_EE_TS = an_EE_TS[an_EE_TS['county'] == county] # Filter Grant
an_EE_TS['SF_year'] = SF_year

########################################################################################

# output_dir = "/data/hydro/users/Hossein/remote_sensing/02_Regularized_TS/"

# os.makedirs(output_dir, exist_ok=True)
########################################################################################

if (indeks == "EVI"):
    an_EE_TS = rc.initial_clean_EVI(an_EE_TS)
else:
    an_EE_TS = rc.initial_clean_NDVI(an_EE_TS)

an_EE_TS.head(2)

###
### List of unique polygons
###
polygon_list = an_EE_TS['ID'].unique()
print(len(polygon_list))

########################################################################################
###
###  initialize output data. all polygons in this case
###  will have the same length. 
###  9 steps in the first three months, followed by 36 points in the full year,
###  9 months in the last year
###
reg_cols = ['ID', 'Acres', 'county', 'CropGrp', 'CropTyp',
            'DataSrc', 'ExctAcr', 'IntlSrD', 'Irrigtn', 'LstSrvD', 'Notes',
            'RtCrpTy', 'Shap_Ar', 'Shp_Lng', 'TRS', 'image_year', 
            'SF_year', 'doy', indeks]

nrows = 54 * len(polygon_list)
output_df = pd.DataFrame(data = None,
                         index = np.arange(nrows), 
                         columns = reg_cols)
########################################################################################

counter = 0

for a_poly in polygon_list:
    if (counter): # % 100 == 0
        print (counter)
    
    curr_field = an_EE_TS[an_EE_TS['ID']==a_poly].copy()
    ################################################################
    # Sort by DoY (sanitary check)
    curr_field.sort_values(by=['image_year', 'doy'], inplace=True)
    
    curr_field = rc.correct_timeColumns_dataTypes(curr_field)
    curr_field.reset_index(drop=True, inplace=True)
    
    print ("print(curr_field.shape")
    print(curr_field.shape)
    print ("__________________________________________")
    ################################################################
    regularized_TS = rc.regularize_movingWindow_windowSteps_18Months(curr_field, \
                                                                     SF_yr = SF_year, \
                                                                     idks = indeks, \
                                                                     window_size = 10)
    print(regularized_TS.shape)

    ################################################################
    row_pointer = 54 * counter
    output_df[row_pointer: row_pointer+54] = regularized_TS.values
    counter += 1



regularized_TS.values.shape
output_df[row_pointer: row_pointer+54].shape
row_pointer
output_df.shape
output_df.head(2)
print (time.strftime('%Y-%m-%d', time.localtime(a_df_1.system_start_time.iloc[0])))
print (a_df_1.system_start_time.iloc[0])
print (time.strftime('%Y-%m-%d', time.localtime(a_df_1.system_start_time.iloc[0])))


print ("Convert Epoch to datetime format")
print (datetime.datetime.fromtimestamp(a_df_1.system_start_time.iloc[0]))

# Convert Epoch to DoY
print ("___________________________________________")
print ("")
print ("Convert Epoch to DoY")
print ( (datetime.datetime.fromtimestamp(a_df_1.system_start_time.iloc[0])).timetuple().tm_yday  )

print ("___________________________________________")
print ("")
print ("difference number of days")
print ((date(2003,11,22) - date(2002,10,20)).days)
time.localtime(a_df_1.system_start_time.iloc[0])
# datetime.datetime(2016, 1, 1) + datetime.timedelta(275 - 1)
# im_yr_sotred = a_df_1.copy()
# epoch_sorted = a_df_1.copy()
# im_yr_sotred.sort_values(by=['image_year', 'doy'], inplace=True)
# epoch_sorted.sort_values(by=['system_start_time'], inplace=True)
# epoch_sorted.to_csv (r'/Users/hn/Desktop/test/epoch_sorted.csv', index = True, header=True)
# im_yr_sotred.to_csv (r'/Users/hn/Desktop/test/im_yr_sotred.csv', index = True, header=True)
# a_df_1.to_csv (r'/Users/hn/Desktop/test/a_df_1.csv', index = True, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2519663683211283, 'fraction_numerical': 0.016815839435855708, 'mean_word_length': 5.218381112984823, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '141447', 'license': 'MIT', 'n_tokens_mistral': 2538, 'n_tokens_neox': 2400, 'n_words': 417, 'path': 'remote_sensing/python/Local_Jupyter_NoteBooks/scratches_to_experiment/moving_10_day_window_2Yrs.ipynb', 'repo_name': 'HNoorazar/Kirti', 'size': '7374'}