<commit_before>import pandas as pd, numpy as np, os, sys, string
from utils import misc

instore = pd.HDFStore(os.path.join(misc.data_dir(),'baseyeardata.h5'))
outstore = pd.HDFStore(os.path.join(misc.data_dir(),'baseyeardata_condensed.h5'))

for tblname in instore.keys():
    tblname = string.replace(tblname,'/','')
    if tblname in ['nodes']: continue
    print "\n\nCondensing: " + tblname
    tbl = instore[tblname]
    newtbl = pd.DataFrame(index=tbl.index)
    for colname in instore[tblname].columns:
        if not os.popen('find | grep "\.py$" | grep -v scripts | xargs grep %s' % colname).read():
            del tbl[colname]
        elif colname in ['geom','txt_geom','nodeid']:
            del tbl[colname]
        else: 
            if tbl[colname].dtype == np.float64: newtbl[colname] = tbl[colname].astype('float32') 
            elif tbl[colname].dtype == np.int64: newtbl[colname] = tbl[colname].astype('int32')
            else: newtbl[colname] = tbl[colname]
    print newtbl.columns
    print newtbl
    outstore[tblname] = newtbl<commit_msg>Update condense_h5.py<commit_after>import pandas as pd, numpy as np, os, sys, string
from utils import misc

instore = pd.HDFStore(os.path.join(misc.data_dir(),'baseyeardata.h5'))
outstore = pd.HDFStore(os.path.join(misc.data_dir(),'baseyeardata_condensed.h5'))

for tblname in instore.keys():
    tblname = string.replace(tblname,'/','')
    if tblname in ['nodes']: continue
    print "\n\nCondensing: " + tblname
    tbl = instore[tblname]
    newtbl = pd.DataFrame(index=tbl.index)
    for colname in instore[tblname].columns:
        if not os.popen('find | grep "\.json$" | grep -v scripts | xargs grep %s' % colname).read():
            del tbl[colname]
        elif colname in ['geom','txt_geom','nodeid']:
            del tbl[colname]
        else: 
            if tbl[colname].dtype == np.float64: newtbl[colname] = tbl[colname].astype('float32') 
            elif tbl[colname].dtype == np.int64: newtbl[colname] = tbl[colname].astype('int32')
            else: newtbl[colname] = tbl[colname]
    print newtbl.columns
    print newtbl
    outstore[tblname] = newtbl
{'fraction_non_alphanumeric': 0.14626168224299066, 'fraction_numerical': 0.009813084112149532, 'mean_word_length': 3.256461232604374, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1422487', 'n_tokens_mistral': 784, 'n_tokens_neox': 739, 'n_words': 160}