<jupyter_start><jupyter_text><jupyter_code>try:
    from google.colab import drive
    drive.mount('/content/drive', force_remount=True)
    COLAB = True
    print("Note: using Google CoLab")
    %tensorflow_version 2.x
except:
    print("Note Google CoLab")
    COLAB = False


!pip install pytz
!pip install https://github.com/neuropsychology/neurokit/zipball/master
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import pytz
import os
import matplotlib.pyplot as plt
import neurokit2 as nk
from scipy import signal
import csv
def condition(value):
    if value <= 13:
        return 0 #minimal
    elif (value>=14) and (value<=19):
        return 1 #mild
    elif (value>=20) and (value<=28):
        return 2 #moderate
    elif value >= 29:
        return 3 #severe


def timeconversion(value):
  # print(pytz.all_timezones)
  tz = pytz.timezone('Asia/Shanghai')
  return datetime.fromtimestamp(value/1000, tz=tz).strftime('%Y-%m-%d %H:%M:%S')

def endtime(value):
  # print(pytz.all_timezones)
  tz = pytz.timezone('Asia/Shanghai')
  return datetime.fromtimestamp((value-1.8e+6)/1000, tz=tz).strftime('%Y-%m-%d %H:%M:%S')

DATA_PATH = "/content/drive/MyDrive/ACMV1/Stress&Mood/trait.xlsx - Pre-test.csv"

df = pd.read_csv(DATA_PATH)
df = df[['Participant ID', 'BDI-II_1', 'BDI-II_2', 'BDI-II_3', 'BDI-II_4', 'BDI-II_5', 'BDI-II_6', 'BDI-II_7', 'BDI-II_8', 'BDI-II_9', 'BDI-II_10', 'BDI-II_11', 'BDI-II_12',
         'BDI-II_13', 'BDI-II_14', 'BDI-II_15', 'BDI-II_16', 'BDI-II_17', 'BDI-II_18', 'BDI-II_19', 'BDI-II_20', 'BDI-II_21']]

df['BDI-II-Score'] = df[['BDI-II_1', 'BDI-II_2', 'BDI-II_3', 'BDI-II_4', 'BDI-II_5', 'BDI-II_6', 'BDI-II_7', 'BDI-II_8', 'BDI-II_9', 'BDI-II_10', 'BDI-II_11', 'BDI-II_12',
         'BDI-II_13', 'BDI-II_14', 'BDI-II_15', 'BDI-II_16', 'BDI-II_17', 'BDI-II_18', 'BDI-II_19', 'BDI-II_20', 'BDI-II_21']].sum(axis=1)


print(len(df['Participant ID'].unique()))
df = df[~df.isin([np.nan, np.inf, -np.inf]).any(1)].astype(np.float64)

df['BDI_Numeric'] = df['BDI-II-Score'].apply(condition)

display(df)
df.to_csv("/content/drive/MyDrive/ACMV1/Stress&Mood/BDI_Scores_Formatted.csv")

bdi_pretest_df = pd.read_csv("/content/drive/MyDrive/ACMV1/Stress&Mood/BDI_Scores_Formatted.csv")
display(bdi_pretest_df)

#I CAN READ EXCEL!
esm_df =  pd.read_excel("/content/drive/MyDrive/ACMV1/Stress&Mood/Psychol_Rec/ESM.xlsx")

esm_df['PPGEnd'] = esm_df[' StartTimeStamp '].apply(timeconversion)
esm_df['PPGStart'] = esm_df[' StartTimeStamp '].apply(endtime)

print(esm_df['Participant ID'].value_counts())

print(len(esm_df[esm_df['Participant ID'] == 1001])

)
display(esm_df)



folder_1 = "/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec1/Physiol_Rec"
folder_2 = "/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec2/Physiol_Rec"
folder_3 = "/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec3/Physiol_Rec"

def get_files(folder):
    list = []
    for the_file in os.listdir(folder):
        # the_file = the_file.split(".")
        list.append(the_file)
    mylist = set(list)
    print(len(mylist))
    return sorted(mylist)

def plotme(data):
        plt.figure(figsize=(20,8))
        plt.subplot(211)
        plt.plot(data)
        plt.show()
  
def timeslice(time):
  year = time[0:4]
  month = time[4:6]
  day = time[6:8]
  hour = time[8:10]
  min = time[10:12]
  sec = time[12:14]
  ms = time[14:16]

  return (datetime(*map(int, [year, month, day, hour, min, sec])))
  

  



patient0 = get_files("/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec1/Physiol_Rec/1001")

for x in patient0:

        print(x)

        if ('PPG' in x) and ('gsheet' not in x):
          ppg_data_df = pd.read_csv("/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec1/Physiol_Rec/1001/{}".format(x))
          display(ppg_data_df.head())
          signal_mine = np.array(ppg_data_df["PPG"])
          the_file = x.split(".")[0]

          

          time1 = the_file.split("_")[0]
          time2= the_file.split("_")[1]

          print(time1)
          print(time2)

          timeslice(time1)
          timeslice(time2)

        # interval = 120
        # i = 0

        # duration = len(signal_mine)/20.0
        # print("Duration", duration)
        # time = np.arange(0,duration,1/20.0)
        # no_1_minute_intervals = int(((duration)/60))
        # print("No of intervals:", no_1_minute_intervals)


        # my_signal = signal_mine[(interval*i*20):((i+1)*interval*20)]   
        # my_signal = my_signal[my_signal > 100]

        # time = np.arange(0,len(my_signal),1/20.0)
        # plt.plot(time,my_signal)
        # plt.xlabel('Time [s]')
        # plt.ylabel('Amplitude')
        # plt.show()







# ppg_set_1 = get_files(folder_1)
# ppg_set_2 = get_files(folder_2)
# ppg_set_3 = get_files(folder_3)

# for x in ppg_set_1:
#       file_path = os.path.join(folder_1, x)
#       print(file_path)
#       user_data = get_files(file_path)
#       for y in user_data:
#          data_path = os.path.join(file_path, y)
#          print(data_path)
#          print(y)
#          name = y.split(".")[0]
#          start_watch = name.split("_")[0]
#          end_watch = name.split("_")[1]
#          try:
#             signal_type = name.split("_")[2]
#             print("Signal Type {}, with times = {} to {}".format(signal_type, start_watch, end_watch))
#             ppg_data_df = pd.read_csv(data_path)
#             display(ppg_data_df.head())

#             signal = np.array(ppg_data_df["PPG"])

      

#             plotme(signal)
#             break
#          except:
#                 print("Summary file")


ppg_data_df = pd.read_csv("/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec1/Physiol_Rec/1001/20191124092457_20191125220645_PPG.csv")
display(ppg_data_df.head())
signal = np.array(ppg_data_df["PPG"])
calc_hrv_metrics_ppg(signal, "mexico")
import re 

def timeslice_ppg(time):
  year = time[0:4]
  month = time[4:6]
  day = time[6:8]
  hour = time[8:10]
  min = time[10:12]
  sec = time[12:14]
  ms = time[14:16]

  return (datetime(*map(int, [year, month, day, hour, min, sec])))
  
def timeslice_ppg_v2(time):
  year = time[0:4]
  month = time[4:6]
  day = time[6:8]
  hour = time[8:10]
  min = time[10:12]
  sec = time[12:14]
  ms = time[14:16]

  return (datetime(*map(int, [year, month, day, hour, min, sec])))
  
def date_time_formatted(date):

  try:
    int(date[:4])
    date = date.replace('-',',')
    date = date.replace(':',',')
    date = date.replace(' ',',')
    date = date.replace(',','')
    return timeslice_ppg(date)
  except:
    date = datetime.strptime(date, '%d-%b-%Y %H:%M:%S')
    return date


date1 = '2019-11-25 22:11:40'
date2 = '26-Nov-2019 21:42:23'

print(date_time_formatted(date1) > date_time_formatted(date2))



# print(re.sub([\w,:],"", str(date1)))




csvdata = []
csvdata.append(['Patient','PPG', 'Milliseconds','Timestamp', 'StartTime', 'EndTime', 'Place','People','Activity', 'TIPI-C_1',	'TIPI-C_2',	'TIPI-C_3',	'TIPI-C_4',	'TIPI-C_5',	'PANAS_1', 'PANAS_2','PANAS_3', 'PANAS_4', 'PANAS_5',	'PANAS_6', 'PANAS_7',	'PANAS_8', 'PANAS_9',	'PANAS_10',	'Valence', 'Arousal'])

ppg_signals_only = []
time_stamps_normal = []
time_stamps = []
values = []
gsr_signals_only = []

break_counter = 0


for index, row in esm_df.iterrows():
  print("Patient:", row['Participant ID'])


  patient = get_files("/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec1/Physiol_Rec/{}".format(row['Participant ID']))

  exp_start = row['PPGStart']
  exp_end = row['PPGEnd']


  exp_start = date_time_formatted(exp_start)
  exp_end = date_time_formatted(exp_end)

  if row['Participant ID'] > 1001:
    break

  for x in patient:

    # print('break counter', break_counter)
    # if break_counter > 100:
    #   break
    
    if ('PPG' in x) and ('gsheet' not in x):
      ppg_data_df = pd.read_csv("/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec1/Physiol_Rec/{}/{}".format(row['Participant ID'],(x)))
    
      the_file = x.split(".")[0]
      time1 = the_file.split("_")[0]
      time2= the_file.split("_")[1]
      ppg_start = timeslice(time1)
      ppg_end = timeslice(time2)

      print("Exp Start",exp_start)
      print("Exp End",exp_end)
      print("PPG Start", ppg_start)
      print("PPG End", ppg_end)

  
      ppg_data_df['csv_time_PPG'] = ppg_data_df['csv_time_PPG'].apply(date_time_formatted)
      # display(ppg_data_df)


      print("PPG Random 1", (ppg_data_df['csv_time_PPG'].iloc[42069]))

      # print(ppg_data_df['csv_time_PPG'].iloc[42069] >= exp_start)
      # print(ppg_data_df['csv_time_PPG'].iloc[42069] <= exp_end)
      

      mexico = ppg_data_df.loc[(ppg_data_df['csv_time_PPG'] >= exp_start) & (ppg_data_df['csv_time_PPG'] <= exp_end)]
      display(mexico)


      if len(mexico) > 0:
        ppg_signals_only.append(np.array(mexico['PPG'], dtype=np.float))
        time_stamps_normal.append(np.array(mexico['csv_time_PPG'], np.float))
        time_stamps.append(np.array(mexico['csv_time_PPG'].map(lambda x: x.timestamp()), dtype=np.float))
        values.append([row['Participant ID'],exp_start,exp_end, row['Place of the event'],row['Participating people'],	row['Activity type'], row['TIPI-C_1'],	row['TIPI-C_2'],	row['TIPI-C_3'],	row['TIPI-C_4'], row['TIPI-C_5'],	row['PANAS_1'], row['PANAS_2'], row['PANAS_3'], row['PANAS_4'], row['PANAS_5'], row['PANAS_6'], row['PANAS_7'], row['PANAS_8'], row['PANAS_9'], row['PANAS_10'],row['Valence'], row['Arousal']])
    
    if ('GSR' in x) and ('gsheet' not in x):
      gsr_data_df = pd.read_csv("/content/drive/MyDrive/ACMV1/Stress&Mood/Physiol_Rec1/Physiol_Rec/{}/{}".format(row['Participant ID'],(x)))
    
      the_file = x.split(".")[0]
      time1 = the_file.split("_")[0]
      time2= the_file.split("_")[1]
      ppg_start = timeslice(time1)
      ppg_end = timeslice(time2)

      print("Exp Start",exp_start)
      print("Exp End",exp_end)
      print("GSR Start", ppg_start)
      print("GSR End", ppg_end)

  
      gsr_data_df['csv_time_GSR'] = gsr_data_df['csv_time_GSR'].apply(date_time_formatted)
      # display(gsr_data_df)


      print("GSR Random 1", (gsr_data_df['csv_time_GSR'].iloc[42069]))

      # print(gsr_data_df['csv_time_PPG'].iloc[42069] >= exp_start)
      # print(gsr_data_df['csv_time_PPG'].iloc[42069] <= exp_end)
      

      guatemala = gsr_data_df.loc[(gsr_data_df['csv_time_GSR'] >= exp_start) & (gsr_data_df['csv_time_GSR'] <= exp_end)]
      display(guatemala)


      if len(guatemala) > 0:
        gsr_signals_only.append(np.array(guatemala['GSR'], dtype=np.float))
      
    # break_counter = break_counter + 1
  

# print(ppg_signals_only)
# print(time_stamps_normal)
# print(time_stamps)
# print(values)


df_signals = pd.DataFrame(list(zip(np.array(ppg_signals_only, dtype=np.float), np.array(time_stamps_normal,dtype=np.float), np.array(time_stamps, dtype=np.float), np.array(gsr_signals_only, dtype=np.float))), columns=['PPG', 'Milliseconds','Timestamp', 'GSR'])   

new_df = pd.DataFrame(values, columns = ['Patient','StartTime', 'EndTime', 'Place','People','Activity', 'TIPI-C_1',	'TIPI-C_2',	'TIPI-C_3',	'TIPI-C_4',	'TIPI-C_5',	'PANAS_1', 'PANAS_2','PANAS_3', 'PANAS_4', 'PANAS_5',	'PANAS_6', 'PANAS_7',	'PANAS_8', 'PANAS_9',	'PANAS_10',	'Valence', 'Arousal'])


frames = [df_signals, new_df]

result = pd.concat(frames, axis=1)
display(result)

print(result.dtypes)
import pickle
result.to_pickle('/content/drive/MyDrive/ACMV1/Stress&Mood/physiorec_p1_pandas_v1.pkl')


# with open('/content/drive/MyDrive/ACMV1/Stress&Mood/physiorec_pt_1.pkl', "wb") as fOut:
#     pickle.dump({'PPG': result['PPG'], 'GSR': result['GSR'], 'Timestamps': result['Timestamp'], 'Patient':result['Patient'], 'StartTime':result['StartTime'], 'EndTime':['EndTime'], 
#                  'PANAS_1': result['PANAS_1'],'PANAS_2':result['PANAS_2'], 'PANAS_3': result['PANAS_3'], 'PANAS_4': result['PANAS_4'], 'PANAS_5': result['PANAS_5'], 
#                  'PANAS_6': result['PANAS_6'], 'PANAS_7':result['PANAS_7'], 'PANAS_8':result['PANAS_8'], 
#                  'PANAS_9':	result['PANAS_10'], 'Valence':result['Valence'], 'Arousal':result['Arousal']}, fOut, protocol=pickle.HIGHEST_PROTOCOL)
DATA_PATH = "/content/drive/MyDrive/ACMV1/Stress&Mood/physiorec_pandas_1.pkl"

signal_df_ph1 = pd.read_pickle(DATA_PATH)


display(signal_df_ph1)


signal_df_ph1.rename(columns={'PANAS_1': 'upset', 'PANAS_2': 'hostile','PANAS_3': 'alert', 'PANAS_4': 'ashamed','PANAS_5': 'inspired', 'PANAS_6': 'nervous','PANAS_7': 'determined', 'PANAS_8': 'attentive','PANAS_9': 'afraid', 'PANAS_10': 'active'}, inplace=True)


signal_ppg = np.array(signal_df_ph1['GSR'].iloc[0])


print(len(signal_ppg))

print(signal_ppg)

plotme(signal_ppg)


# print(esm_df['PPGEnd'][esm_df['Participant ID'] == 1001])

# print(signal_df_ph1['EndTime'][signal_df_ph1['Patient'] == 1001])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1832007487131493, 'fraction_numerical': 0.03743565746373421, 'mean_word_length': 3.9875534811357447, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '567928', 'license': 'MIT', 'n_tokens_mistral': 5686, 'n_tokens_neox': 5357, 'n_words': 756, 'path': 'ACM_PP_v1.ipynb', 'repo_name': 'prophets-of-eternity/summer-2021-v3', 'size': '12822'}