<jupyter_start><jupyter_text>Load<jupyter_code>import os
arr = os.listdir('data/')
for i in arr:
    if i not in ['all.csv']:
        dfz[i[:-4]]=pd.read_csv('data/'+i)
        print(i)
stationc=station.set_index(0)[2]
for z in dfz:
    dfz[z]=dfz[z].reset_index()
stationx={}
for z in dfz:
    stationx[stationc.loc[int(z)]]={'id':z,'yrs':len(dfz[z]['year'].unique())}
dq=pd.DataFrame(stationx).T.reset_index().set_index('yrs').sort_index(ascending=False)
dq.head(5)
dds=[]
indicator='temp_avg'
for a in dq.head(500)['id'].values:
    dw=dfz[a]
    tmean=dw.groupby(['month','hour']).mean()[[indicator]]
    tmean.columns=['temp_mean']
    dw=dw.loc[1980:]
    dw=dw.loc[:2010]
    tmean80=dw.groupby(['month','hour']).mean()[[indicator]]
    tmean80.columns=['temp_mean80']
    dc=dfz[a].groupby(['year','month','hour']).mean()[[indicator]].join(tmean).join(tmean80)
    dc['temp_delta']=dc[indicator]-dc['temp_mean']
    dc['temp_delta80']=dc[indicator]-dc['temp_mean80']
    dd=dc.groupby(['year']).mean()
    dd=np.round(dd,1)
    dd['varos']=stationc[int(a)]
    dd['value']=1
    dds.append(dd)
ddb=pd.concat(dds)
ddb.to_csv('stripes/geo.csv')
dgeo=ddb.reset_index().set_index('varos').join(station.set_index(2))
dgeo[dgeo[3]=='HUNGARY'].to_csv('stripes/geotest.csv')
dds=[]
indicator='temp_avg'
# clean_months=[1,4,7,10]
clean_months=[1,2,3,4,5,6,7,8,9,10,11,12]
# clean_hours=[0,6,12,18]
clean_hours=[0,3,6,12,15,18,21]
# clean_hours=[6]
clean_slice=[(i,j) for i in clean_months for j in clean_hours]
for a in dq['id'].values:
    dw=dfz[a].groupby(['month','hour']).mean().loc[clean_slice]
    if dw[indicator].count()==len(clean_hours)*len(clean_months):
        
        dc=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index()
        dx=dc.groupby(['year']).nunique()
        full_data_years=dx[((dx['month']==len(clean_months))&(dx['hour']==len(clean_hours)))].index
        tmean=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean.columns=['temp_mean']

        dw=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index().set_index(['year']).loc[full_data_years].reset_index()
        tmean_full_years=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean_full_years.columns=['temp_mean_full_years']

        dc=dc.groupby(['year','month','hour']).mean()[[indicator]].loc[full_data_years].join(tmean).join(tmean_full_years)
        dc['temp_delta']=dc[indicator]-dc['temp_mean']     
        dc['temp_delta_full_years']=dc[indicator]-dc['temp_mean_full_years']     

        if ((2018 in dw['year'].unique()) and (2010 in dw['year'].unique())):
            tmean_last10=dw[dw['year']>2009].groupby(['month','hour']).mean()[[indicator]]
            tmean_last10.columns=['temp_mean_last10']
            dc=dc.join(tmean_last10)
            dc['temp_delta_last10']=dc[indicator]-dc['temp_mean_last10']
        
        dd=dc.groupby(['year']).mean()
        dd=np.round(dd,1)
        dd['varos']=stationc[int(a)]
        dd['value']=1
        dds.append(dd)
ddb=pd.concat(dds)
dgeo=ddb.reset_index().set_index('varos').join(station.set_index(2))
dgeo[dgeo[3]=='HUNGARY'].to_csv('stripes/hu.csv')
dgeo[dgeo[3]=='ROMANIA'].to_csv('stripes/ro.csv')
dgeo.to_csv('stripes/huro.csv')
dds=[]
indicator='temp_avg'
clean_months=[1,5,9]
# clean_months=[1,2,3,4,5,6,7,8,9,10,11,12]
# clean_hours=[0,6,12,18]
# clean_hours=[0,3,6,12,15,18,21]
clean_hours=[6]
clean_slice=[(i,j) for i in clean_months for j in clean_hours]
for a in dq['id'].values:
    dw=dfz[a].groupby(['month','hour']).mean().loc[clean_slice]
    if dw[indicator].count()==len(clean_hours)*len(clean_months):
        
        dc=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index()
        dx=dc.groupby(['year']).nunique()
        full_data_years=dx[((dx['month']==len(clean_months))&(dx['hour']==len(clean_hours)))].index
        tmean=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean.columns=['temp_mean']

        dw=dfz[a].set_index(['month','hour']).loc[clean_slice].reset_index().set_index(['year']).loc[full_data_years].reset_index()
        tmean_full_years=dw.groupby(['month','hour']).mean()[[indicator]]
        tmean_full_years.columns=['temp_mean_full_years']

        dc=dc.groupby(['year','month','hour']).mean()[[indicator]].loc[full_data_years].join(tmean).join(tmean_full_years)
        dc['temp_delta']=dc[indicator]-dc['temp_mean']     
        dc['temp_delta_full_years']=dc[indicator]-dc['temp_mean_full_years']     

        if ((2018 in dw['year'].unique()) and (2010 in dw['year'].unique())):
            tmean_last10=dw[dw['year']>2009].groupby(['month','hour']).mean()[[indicator]]
            tmean_last10.columns=['temp_mean_last10']
            dc=dc.join(tmean_last10)
            dc['temp_delta_last10']=dc[indicator]-dc['temp_mean_last10']
        
        dd=dc.groupby(['year']).mean()
        dd=np.round(dd,1)
        dd['varos']=stationc[int(a)]
        dd['value']=1
        dds.append(dd)
ddb=pd.concat(dds)
dgeo=ddb.reset_index().set_index('varos').join(station.set_index(2))
dgeo[dgeo[3]=='HUNGARY'].to_csv('stripes/hu_mini.csv')
dgeo[dgeo[3]=='ROMANIA'].to_csv('stripes/ro_mini.csv')
dgeo.to_csv('stripes/huro_mini.csv')
station[station[2]=='DEBRECEN']
namer={"AGARD":"Agárd",
"AUREL VLAICU":"Bukarest - Aurel Vlaicu",
"BACAU":"Bákó",
"BAJA":"Baja",
"BEKESCSABA":"Békéscsaba",
"BOBOC AIR BASE":"Boboc légi bázis",
"BOTOSANI":"Botoșani",
"BUDAPEST/PESTSZENTLORINC":"Budapest - Pestszentlőrinc",
"BUZAU":"Buzău",
"CALARASI":"Călărasi",
"CAMPIA TURZII":"Aranyosgyéres",
"CARANSEBES":"Káránszebes",
"CATALOI":"Tulcea - Cataloi",
"<NAME>":"Csalhó",
"<NAME>":"Kolozsvár",
"CONSTANTA":"Konstanca",
"DEBRECEN":"Debrecen",
"DEVA":"Déva",
"DROBETA TURNU SEVERIN":"Szörényvár",
"EGER":"Eger",
"FERIHEGY":"Budapest - Ferihegy",
"GALATI":"Galac",
"GYOR":"Győr",
"<NAME>":"Bukarest - <NAME>",
"IASI":"Jászvásár",
"JOSVAFO":"Jósvafő",
"KECSKEMET":"Kecskemét",
"KEKESTETO":"Kékestető",
"<NAME>":"Csíkszereda",
"<NAME>":"Konstanca - <NAME>",
"MISKOLC":"Miskolc",
"MOSONMAGYAROVAR":"Mosonmagyaróvár",
"NAGYKANIZSA":"Nagykanizsa",
"NYIREGYHAZA":"Nyíregyháza",
"OC<NAME>":"Aknasugatag",
"ORADEA":"Nagyvárad",
"PAKS":"Paks",
"PAPA":"Pápa",
"PECS SOUTH":"Pécs",
"POROSZLO":"Poroszló",
"<NAME>":"<NAME>",
"ROSIORII DE VEDE":"Roșiorii De Vede",
"SARMELLEK":"Balaton - Sármellék",
"SAT<NAME>":"Szatmár",
"SIBIU":"Nagyszeben",
"SIOFOK":"Siófok",
"SOPRON":"Sopron",
"<NAME>":"Suceava - Ște<NAME>",
"SULINA":"Sulina",
"SZECSENY":"Szécsény",
"SZEGED (AUT)":"Szeged",
"SZENTGOTTHARD/FARKASFA":"Szentgotthárd",
"SZOLNOK":"Szolnok",
"SZOMBATHELY ARPT / VAS":"Szombathely",
"TAT":"Tát",
"TATA":"Tata",
"TAUTII MAGHERAUS":"Nagybánya - Miszmogyorós",
"TRAIAN VUIA":"Temesvár - Traian Vuia",
"VARFU OMU":"Bucsecs - Omu csúcs",
"VESZPREM/SZENTKIRALYSZABADJA":"Veszprém",
"VIDRASAU":"Marosvásárhely - Vidrátszeg",
"ZAHONY":"Záhony",
"ZALAEGERSZEG/ANDRASHIDA":"Zalaegerszeg"}
import json
open('namer.json','w').write(json.dumps(namer))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.23309557774607703, 'fraction_numerical': 0.023395149786019972, 'mean_word_length': 6.025050100200401, 'pattern_counts': {'":': 63, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '204045', 'license': 'MIT', 'n_tokens_mistral': 3197, 'n_tokens_neox': 3020, 'n_words': 267, 'path': 'klima/archive/climate-daily-data1.ipynb', 'repo_name': 'szekelydata/szekelydata', 'size': '7103'}