<jupyter_start><jupyter_text>**!!! 16G memory required at least, 64G recommended**<jupyter_code>dfs=[]
for i in ro:
    df=pd.read_csv(p+'raw/ro/'+i+'dat.txt',delimiter= '\s+')
    dfs.append(df)
    print(i)
dfz=pd.concat(dfs)
dfs=None #free memory
df=None #free memory
dfz.head()
#!!!! DO NOT DO THIS
#dfz=dfz.drop_duplicates()
dfz['time']=pd.to_datetime(dfz['YR--MODAHRMN'],format='%Y%m%d%H%M')
dfz['year']=dfz['time'].dt.year
dfz['month']=dfz['time'].dt.month
dfz['day']=dfz['time'].dt.day
dfz['hour']=dfz['time'].dt.hour
#keep only months with at least 6 days to avoid anomalies (20%)
filt=dfz.groupby(['USAF','year','month'])[['day']].nunique()
filt2=filt[filt>5].dropna()
#keep only years with at least 3 months to avoid anomalies (20%)
filt3=filt2.reset_index().groupby(['USAF','year'])[['month']].nunique()
filt4=filt3[filt3>3].dropna()
filt4.head()
filt4.index.unique(0)
for i in filt4.index.unique(0):
    d=df_avg2.loc[i].loc[filt4.loc[i].index.unique()]
    dfz[i]=d
    d.to_csv('data/'+str(i)+'.csv')
    print(i)<jupyter_output>119000
127560
127660
127720
127860
128050
128120
128150
128220
128250
128300
128305
128310
128360
128380
128390
128400
128430
128460
128470
128510
128550
128600
128603
128605
128660
128700
128820
128920
129100
129150
129200
129220
129250
129255
129300
129320
129350
129400
129410
129420
129500
129600
129700
129820
129920
150000
150001
150002
150010
150040
150070
150090
150100
150105
150140
150150
150200
150230
150235
150250
150320
150330
150400
150410
150420
150440
150470
150520
150550
150560
150630
150690
150730
150750
150800
150830
150850
150880
150890
150900
150940
150950
150990
151070
151080
151090
151110
151130
151170
151180
151190
151200
151205
151230
151240
151270
151320
151340
151360
151380
151400
151430
151450
151455
151480
151500
151540
151580
151590
151600
151620
151630
151650
151680
151700
151740
151790
151820
151840
151890
151940
151970
151990
152000
152005
152040
152060
152080
152090
152120
152150
152170
152190
152210
152300
152310
152350
152380
152410
152450
152470
152540[...]<jupyter_text>Load<jupyter_code>import os
arr = os.listdir('data/')
for i in arr:
    if i not in ['all.csv']:
        dfz[i[:-4]]=pd.read_csv('data/'+i)
        print(i)
stationc=station.set_index(0)[2]
for z in dfz:
    dfz[z]=dfz[z].reset_index()
stationx={}
for z in dfz:
    stationx[stationc.loc[int(z)]]={'id':z,'yrs':len(dfz[z]['year'].unique())}
dq=pd.DataFrame(stationx).T.reset_index().set_index('yrs').sort_index(ascending=False)
dq.head(5)
dds=[]
indicator='temp_avg'
for a in dq.head(500)['id'].values:
    dw=dfz[a]
    tmean=dw.groupby(['month','hour']).mean()[[indicator]]
    tmean.columns=['temp_mean']
    dw=dw.loc[1980:]
    dw=dw.loc[:2010]
    tmean80=dw.groupby(['month','hour']).mean()[[indicator]]
    tmean80.columns=['temp_mean80']
    dc=dfz[a].groupby(['year','month','hour']).mean()[[indicator]].join(tmean).join(tmean80)
    dc['temp_delta']=dc[indicator]-dc['temp_mean']
    dc['temp_delta80']=dc[indicator]-dc['temp_mean80']
    dd=dc.groupby(['year']).mean()
    dd=np.round(dd,1)
    dd['varos']=stationc[int(a)]
    dd['value']=1
    dds.append(dd)
ddb=pd.concat(dds)
ddb.to_csv('stripes/geo.csv')
dgeo=ddb.reset_index().set_index('varos').join(station.set_index(2))
dgeo[dgeo[3]=='HUNGARY'].to_csv('stripes/geotest.csv')
dds=[]
indicator='temp_avg'
# clean_months=[1,4,7,10]
clean_months=[1,2,3,4,5,6,7,8,9,10,11,12]
# clean_hours=[0,6,12,18]
clean_hours=[0,3,6,12,15,18,21]
# clean_hours=[6]
clean_slice=[(i,j) for i in clean_months for j in clean_hours]
for a in dq['id'].values:
    dw=dfz[a].groupby(['month','hour']).mean().loc[clean_slice]
    if dw[indicator].count()==len(clean_hours)*len(clean_months):
        
        dc=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index()
        dx=dc.groupby(['year']).nunique()
        full_data_years=dx[((dx['month']==len(clean_months))&(dx['hour']==len(clean_hours)))].index
        tmean=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean.columns=['temp_mean']

        dw=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index().set_index(['year']).loc[full_data_years].reset_index()
        tmean_full_years=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean_full_years.columns=['temp_mean_full_years']

        dc=dc.groupby(['year','month','hour']).mean()[[indicator]].loc[full_data_years].join(tmean).join(tmean_full_years)
        dc['temp_delta']=dc[indicator]-dc['temp_mean']     
        dc['temp_delta_full_years']=dc[indicator]-dc['temp_mean_full_years']     

        if ((2018 in dw['year'].unique()) and (2010 in dw['year'].unique())):
            tmean_last10=dw[dw['year']>2009].groupby(['month','hour']).mean()[[indicator]]
            tmean_last10.columns=['temp_mean_last10']
            dc=dc.join(tmean_last10)
            dc['temp_delta_last10']=dc[indicator]-dc['temp_mean_last10']
        
        dd=dc.groupby(['year']).mean()
        dd=np.round(dd,1)
        dd['varos']=stationc[int(a)]
        dd['value']=1
        dds.append(dd)
ddb=pd.concat(dds)
dgeo=ddb.reset_index().set_index('varos').join(station.set_index(2))
dgeo[dgeo[3]=='HUNGARY'].to_csv('stripes/hu.csv')
dgeo[dgeo[3]=='ROMANIA'].to_csv('stripes/ro.csv')
dgeo.to_csv('stripes/huro.csv')
dds=[]
indicator='temp_avg'
clean_months=[1,5,9]
# clean_months=[1,2,3,4,5,6,7,8,9,10,11,12]
# clean_hours=[0,6,12,18]
# clean_hours=[0,3,6,12,15,18,21]
clean_hours=[6]
clean_slice=[(i,j) for i in clean_months for j in clean_hours]
for a in dq['id'].values:
    dw=dfz[a].groupby(['month','hour']).mean().loc[clean_slice]
    if dw[indicator].count()==len(clean_hours)*len(clean_months):
        
        dc=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index()
        dx=dc.groupby(['year']).nunique()
        full_data_years=dx[((dx['month']==len(clean_months))&(dx['hour']==len(clean_hours)))].index
        tmean=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean.columns=['temp_mean']

        dw=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index().set_index(['year']).loc[full_data_years].reset_index()
        tmean_full_years=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean_full_years.columns=['temp_mean_full_years']

        dc=dc.groupby(['year','month','hour']).mean()[[indicator]].loc[full_data_years].join(tmean).join(tmean_full_years)
        dc['temp_delta']=dc[indicator]-dc['temp_mean']     
        dc['temp_delta_full_years']=dc[indicator]-dc['temp_mean_full_years']     

        if ((2018 in dw['year'].unique()) and (2010 in dw['year'].unique())):
            tmean_last10=dw[dw['year']>2009].groupby(['month','hour']).mean()[[indicator]]
            tmean_last10.columns=['temp_mean_last10']
            dc=dc.join(tmean_last10)
            dc['temp_delta_last10']=dc[indicator]-dc['temp_mean_last10']
        
        dd=dc.groupby(['year']).mean()
        dd=np.round(dd,1)
        dd['varos']=stationc[int(a)]
        dd['value']=1
        dds.append(dd)
ddb=pd.concat(dds)
dgeo=ddb.reset_index().set_index('varos').join(station.set_index(2))
dgeo[dgeo[3]=='HUNGARY'].to_csv('stripes/hu_mini.csv')
dgeo[dgeo[3]=='ROMANIA'].to_csv('stripes/ro_mini.csv')
dgeo.to_csv('stripes/huro_mini.csv')
station[station[2]=='DEBRECEN']
namer={"AGARD":"Agárd",
"AUREL VLAICU":"Bukarest - Aurel Vlaicu",
"BACAU":"Bákó",
"BAJA":"Baja",
"BEKESCSABA":"Békéscsaba",
"BOBOC AIR BASE":"Boboc légi bázis",
"BOTOSANI":"Botoșani",
"BUDAPEST/PESTSZENTLORINC":"Budapest - Pestszentlőrinc",
"BUZAU":"Buzău",
"CALARASI":"Călărasi",
"CAMPIA TURZII":"Aranyosgyéres",
"CARANSEBES":"Káránszebes",
"CATALOI":"Tulcea - Cataloi",
"CEAHLAU TOACA":"Csalhó",
"CLUJ NAPOCA":"Kolozsvár",
"CONSTANTA":"Konstanca",
"DEBRECEN":"Debrecen",
"DEVA":"Déva",
"DROBETA TURNU SEVERIN":"Szörényvár",
"EGER":"Eger",
"FERIHEGY":"Budapest - Ferihegy",
"GALATI":"Galac",
"GYOR":"Győr",
"<NAME>":"Bukarest - <NAME>",
"IASI":"Jászvásár",
"JOSVAFO":"Jósvafő",
"KECSKEMET":"Kecskemét",
"KEKESTETO":"Kékestető",
"MIERCUREA CIUC":"Csíkszereda",
"<NAME>":"Konstanca - <NAME>",
"MISKOLC":"Miskolc",
"MOSONMAGYAROVAR":"Mosonmagyaróvár",
"NAGYKANIZSA":"Nagykanizsa",
"NYIREGYHAZA":"Nyíregyháza",
"OC<NAME>AG":"Aknasugatag",
"ORADEA":"Nagyvárad",
"PAKS":"Paks",
"PAPA":"Pápa",
"PECS SOUTH":"Pécs",
"POROSZLO":"Poroszló",
"<NAME>A":"<NAME>",
"ROSIORII DE VEDE":"Roșiorii De Vede",
"SARMELLEK":"Balaton - Sármellék",
"<NAME>":"Szatmár",
"SIBIU":"Nagyszeben",
"SIOFOK":"Siófok",
"SOPRON":"Sopron",
"STEFAN <NAME>ARE":"Suceava - Ștefan <NAME>",
"SULINA":"Sulina",
"SZECSENY":"Szécsény",
"SZEGED (AUT)":"Szeged",
"SZENTGOTTHARD/FARKASFA":"Szentgotthárd",
"SZOLNOK":"Szolnok",
"SZOMBATHELY ARPT / VAS":"Szombathely",
"TAT":"Tát",
"TATA":"Tata",
"<NAME>":"Nagybánya - Miszmogyorós",
"<NAME>":"Temesvár - Traian Vuia",
"VARFU OMU":"Bucsecs - Omu csúcs",
"VESZPREM/SZENTKIRALYSZABADJA":"Veszprém",
"VIDRASAU":"Marosvásárhely - Vidrátszeg",
"ZAHONY":"Záhony",
"ZALAEGERSZEG/ANDRASHIDA":"Zalaegerszeg"}
import json
open('namer.json','w').write(json.dumps(namer))
df=df.replace({'******':np.nan,'*****':np.nan,'****':np.nan,'***':np.nan,'**':np.nan,'*':np.nan})
df['time']=pd.to_datetime(df['YR--MODAHRMN'],format='%Y%m%d%H%M')
#df=df[['time','USAF','SPD','TEMP','PCP06','SD','MW','AW','W']]
df['TEMP']=(pd.to_numeric(df['TEMP'], errors='coerce')-32)*5/9
df['SPD']=pd.to_numeric(df['SPD'], errors='coerce')*1.61
df['PCP06']=pd.to_numeric(df['PCP06'], errors='coerce')*25.4
df['SD']=pd.to_numeric(df['SD'], errors='coerce')*25.4
dfs['time']=pd.to_datetime(df['YR--MODAHRMN'],format='%Y%m%d%H%M')
dfs['year']=dfs['time'].dt.year
dfs['month']=dfs['time'].dt.month
dfs['day']=dfs['time'].dt.day
dfs['hour']=dfs['time'].dt.hour<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21939930055544127, 'fraction_numerical': 0.10964822053075499, 'mean_word_length': 6.29954954954955, 'pattern_counts': {'":': 63, '<': 20, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '487246', 'license': 'MIT', 'n_tokens_mistral': 4999, 'n_tokens_neox': 4329, 'n_words': 499, 'path': 'nepi/archive/1a_parser_high_res_climate_data.ipynb', 'repo_name': 'szekelydata/szekelydata', 'size': '9796'}