<jupyter_start><jupyter_text>Read meteorological dataFolder with [original data](https://drive.google.com/drive/folders/1me2IpDY3om6IRKv_WMT5W6Qw0r9DI74C)<jupyter_code>import os
import zipfile
import pandas as pd
import glob
import keplergl
import geopandas as gpd
import matplotlib.pyplot as plt
import csv
from datetime import datetime
import re
import numpy as np
from pyproj import Proj
%run Meteo_utils.ipynb
os.listdir('data')<jupyter_output><empty_output><jupyter_text>Automate data extraction from Drive downloads<jupyter_code>path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW'
ziplist = glob.glob(f'{path}/*.zip')
#ziplist = glob.glob(f'data/*.zip')
ziplist
folder = 0
zf = zipfile.ZipFile(f'{ziplist[folder]}')
zf.namelist()[0:10]
### exclude macos files (only if copying data if using Drive directly these don't appear)
r = re.compile('^__MACOSX') # only csv files
exclude = list(filter(r.match, zf.namelist()[0:10])) 
clean = list(set(zf.namelist()[0:10])-set(exclude))
files_by_date = order_meteo_zip(path, folder =folder)
files_by_date
df = read_meteo_csv(path =path,folder = folder,file = files_by_date[0])
df.head()
files_by_date[0].split('/')[-1].replace(" ","")
station = files_by_date[0].split('/')[0].replace(" ","").replace(".","")
all_var = get_unique_variables(files_by_date,path =path,folder = folder)
## Variable names:
# BAT  is  battery
# Not very clear: 'H_M Min','WindMnSpdSclr','WindMnSpdS','WindMnDirUnit'

replace_values ={
# Temperature
    'ATAvg' : ['AT1HrAvg', 'AT5minAvg', 'ATAvg','Temperatura'],
    'ATMin' : ['AT1HrMin', 'AT5minMin','T_Min', 'T Min'],
    'ATMax' : ['AT1HrMax','AT5minMax', 'T_Max', 'T Max'],
 
 # Relative Humidity
    'RHAvg' : ['Humedad','RH5minAvg','RHAvg','RelHumidAvg', 'Humedad R', 'RH5m2015'],
    'RHMin' : ['H_Min','RH5minMin','RelHumidMin','H_M Min'],
    'RHMax' : ['H_Max', 'RH5minMax', 'RelHumidMax','H_R Max'],

 # Wind Speed
    'WSAvg' : ['Velocidad','WS5minAvg','WSAvg','WindMnSpdSclr','WindMnSpdS'],
    'WSMin' : ['WindMinSpdSclr','WS5minMin'],
    'WSMax' : ['WindMaxSpdSclr','WS5minMax'],

 # Wind Direction
    'WDAvg' : ['Direccion', 'WDAvg','WD5minAvg','WindMnDirUnit','Dirección'],
    'WDMin' : ['WD5minMin'],
    'WDMax' : ['WD5minMax','WindMaxDir'],

 # Rain
    'PAvg' : ['Precipitacion','TB1hrAcc','TB1minAcc','TB5minAcc','Precipitación']
}

var_list = ['ATAvg','ATMin','ATMax','RHAvg','RHMin','RHMax', 'WSAvg','WSMin','WSMax','WDAvg','WDMin', 'WDMax','PAvg']
#sorted(var_list)
df = read_meteo_csv(path =path,folder = 0,file = files_by_date[0])
df2 = reformat_df(df=df, replace_values=replace_values)
df2.dtypes
len(files_by_date)<jupyter_output><empty_output><jupyter_text>Process files in loop<jupyter_code>%run Meteo_utils.ipynb
## Variable names:
# BAT  is  battery
# Not very clear: 'H_M Min','WindMnSpdSclr','WindMnSpdS','WindMnDirUnit'
replace_values ={
# Temperature
    'ATAvg' :['AT1HrAvg', 'AT5minAvg', 'ATAvg','Temperatura','Temperatuta','TEMPERATURA AIRE Â°C PROM'],
    'ATMin' : ['AT1HrMin', 'AT5minMin','T_Min', 'T Min','TEMPERATURA AIRE Â°C MIN'],
    'ATMax' : ['AT1HrMax','AT5minMax', 'T_Max', 'T Max','TEMPERATURA AIRE Â°C MAX'],
 
 # Relative Humidity
    'RHAvg' : ['Humedad','RH5minAvg','RHAvg','RelHumidAvg', 'Humedad R', 'RH5m2015','HUMEDAD RELATIVA DEL AIRE % PROM'],
    'RHMin' : ['H_Min','RH5minMin','RelHumidMin','H_M Min','HUMEDAD RELATIVA DEL AIRE % MIN'],
    'RHMax' : ['H_Max', 'RH5minMax', 'RelHumidMax','H_R Max','HUMEDAD RELATIVA DEL AIRE % MAX'],

 # Wind Speed
    'WSAvg' : ['Velocidad','WS5minAvg','WSAvg','WindMnSpdSclr','WindMnSpdS','VIENTO VELOCIDAD m/s INST'],
    'WSMin' : ['WindMinSpdSclr','WS5minMin'],
    'WSMax' : ['WindMaxSpdSclr','WS5minMax'],

 # Wind Direction
    'WDAvg' : ['Direccion', 'WDAvg','WD5minAvg','WindMnDirUnit','Dirección','VIENTO DIRECCION Â° INST'],
    'WDMin' : ['WD5minMin'],
    'WDMax' : ['WD5minMax','WindMaxDir'],

 # Rain
    'PAvg' : ['Precipitacion','TB1hrAcc','TB1minAcc','TB5minAcc','Precipitación','PRECIPITACION mm SUM']
}

var_list = ['ATAvg','ATMin','ATMax','RHAvg','RHMin','RHMax', 'WSAvg','WSMin','WSMax','WDAvg','WDMin', 'WDMax','PAvg']
#sorted(var_list)
path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data/RAW'
ziplist = glob.glob(f'{path}/*.zip')
ziplist
file, folder
path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data'
folder = 9
files_by_date = order_meteo_zip(path=f'{path}/RAW/', folder =folder)
#get_unique_variables(files_by_date,path =f'{path}/RAW/',folder = folder)

for file in range(50, len(files_by_date)):
        print(f'{file}/{len(files_by_date)}')
        df = read_meteo_csv(path =f'{path}/RAW',folder = folder,file = files_by_date[file])    
        df2 = reformat_df(df=df, replace_values=replace_values)
        master_df = master_df.append(df2)
        
export_name = f'{files_by_date[0].split("/")[0].replace(" ","").replace(".","")}'
repeat = list(set([export_name +'.csv']) & set(os.listdir(f'{path}/PROCESSED')))
if len(repeat)>0:
    master_df.to_csv(f'{path}/PROCESSED/{export_name}_{len(repeat)+1}.csv')
else :
    master_df.to_csv(f'{path}/PROCESSED/{export_name}.csv')
df
files_by_date = order_meteo_zip(path=f'{path}/RAW/', folder =8,only_csv=False)
files_by_date
path = '/Users/tamarahuete/Documents/Github_repos/TFM21/data'
ziplist = glob.glob(f'{path}/RAW/*.zip')
for folder in range(9, len(ziplist)):
    
    files_by_date = order_meteo_zip(path=f'{path}/RAW/', folder =folder)
    #get_unique_variables(files_by_date,path =f'{path}/RAW/',folder = folder)

    master_df =pd.DataFrame()
    for file in range(0, len(files_by_date)):
        print(f'{file}/{len(files_by_date)}')
        df = read_meteo_csv(path =f'{path}/RAW',folder = folder,file = files_by_date[file])    
        df2 = reformat_df(df=df, replace_values=replace_values)
        master_df = master_df.append(df2)
        
    export_name = f'{files_by_date[0].split("/")[0].replace(" ","").replace(".","")}'
    repeat = list(set([export_name +'.csv']) & set(os.listdir(f'{path}/PROCESSED')))
    if len(repeat)>0:
        master_df.to_csv(f'{path}/PROCESSED/{export_name}_{len(repeat)+1}.csv')
    else :
        master_df.to_csv(f'{path}/PROCESSED/{export_name}.csv')
%run Meteo_utils.ipynb
file, folder
df = read_meteo_csv(path =f'{path}/RAW',folder = folder,file = files_by_date[file])    
#df2 = reformat_df(df=df, replace_values=replace_values)
df
df2 = reformat_df(df=df, replace_values=replace_values)
folder =8
ziplist = glob.glob(f'{path}/RAW/*.zip')
zf = zipfile.ZipFile(f'{ziplist[folder]}')
if len(df.columns) == 1:
        df = df[df.columns[0]].str.split(',',expand=True)
        if len(df.columns) == 1:
            df = df[df.columns[0]].str.split('.',expand=True)
            df[3] =df[3].astype(str).str.cat(df[4].astype(str),sep=".")
            df.drop(columns=4, inplace=True)
        df = df.iloc[1:]   
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19421427533071667, 'fraction_numerical': 0.01642680622183457, 'mean_word_length': 5.751717369970559, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '435539', 'license': 'MIT', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2798, 'n_words': 415, 'path': 'Read_data.ipynb', 'repo_name': 'tamaramegan/frost_prediction_ml', 'size': '6879'}