<commit_before>from __future__ import division
import re
from itertools import product
import gc

from IPython import embed
import pandas as pd
import numpy as np

particle_vars = [u'pf', u'df', u'vt', u'vr', u'vc']
heat_vars = [u'ef']
momentum_vars = [u'vf']
store_format = 'table'

#'vti_GB', 'dfi_GB', 'vci_GB',
#       'pfi_GB', 'efi_GB',
#       
#       'efe_GB', 'vce_GB', 'pfe_GB',
#       'vte_GB', 'dfe_GB'
             #'chie', 'ven', 'ver', 'vec']
def regime_filter(data, leq, less):
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    bool &= (data['efe_GB'] < less) & (data['efi_GB'] < less)
    bool &= (data['efe_GB'] >= leq) & (data['efi_GB'] >= leq)
    data = data.loc[bool]
    return data

div_bounds = {
        'efeITG_GB_div_efiITG_GB': (0.05, 1.5),
        'pfeITG_GB_div_efiITG_GB': (0.05, 2),
        'efeTEM_GB_div_efiTEM_GB': (0.02, 0.5),
        'pfeTEM_GB_div_efiTEM_GB': (0.01, 0.8)
}
def div_filter(store):
    # This is hand-picked:
    # 0.05 <   efeITG/efiITG    < 1.5
    # 0.05 <   efiTEM/efeTEM    < 2
    # 0.02 < abs(pfeITG/efiITG) < 0.5
    # 0.01 < abs(pfeTEM/efiTEM) < 0.8
    for group in store:
        if isinstance(store, pd.HDFStore):
            group = group[1:]
        pre = len(store[group])
        se = store[group]
        if group in div_bounds:
            low, high = div_bounds[group]
            embed()
        else:
            continue

        store[group] = se.loc[(low < se) & (se < high)]
        print('{:.2f}% of sane {!s:<9} points inside div bounds'.format(np.sum(~store[group].isnull()) / pre * 100, group))


def stability_filter(data):
    for col in data.columns:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] not in heat_vars + particle_vars + momentum_vars:
            print('skipping {!s}'.format(col))
            continue
        if splitted[2] == 'TEM':
            gam_filter = 'tem'
        elif splitted[2] == 'ITG':
            gam_filter = 'itg'
        elif splitted[2] == 'ETG':
            gam_filter = 'elec'
        elif splitted[0] in heat_vars and splitted[1] == 'e':<commit_msg>Added new filter (Jonathan stamp of approval)<commit_after>from __future__ import division
import re
from itertools import product
import gc

from IPython import embed
import pandas as pd
import numpy as np

particle_vars = [u'pf', u'df', u'vt', u'vr', u'vc']
heat_vars = [u'ef']
momentum_vars = [u'vf']
store_format = 'table'

#'vti_GB', 'dfi_GB', 'vci_GB',
#       'pfi_GB', 'efi_GB',
#       
#       'efe_GB', 'vce_GB', 'pfe_GB',
#       'vte_GB', 'dfe_GB'
             #'chie', 'ven', 'ver', 'vec']
def regime_filter(data, leq, less):
    bool = pd.Series(np.full(len(data), True, dtype='bool'), index=data.index)
    bool &= (data['efe_GB'] < less) & (data['efi_GB'] < less)
    bool &= (data['efe_GB'] >= leq) & (data['efi_GB'] >= leq)
    data = data.loc[bool]
    return data

div_bounds = {
    'pfeTEM_GB': (0.02, 20),
    'pfeITG_GB': (0.02, 10),
    'efiTEM_GB': (0.05, np.inf),
    'efeITG_GB_div_efiITG_GB': (0.05, 1.5),
    'pfeITG_GB_div_efiITG_GB': (0.02, 0.6),
    'efiTEM_GB_div_efeTEM_GB': (0.05, 2.0),
    'pfeTEM_GB_div_efeTEM_GB': (0.03, 0.8)
}
def div_filter(store):
    for group in store:
        if isinstance(store, pd.HDFStore):
            group = group[1:]
        pre = len(store[group])
        se = store[group]
        if group in div_bounds:
            low, high = div_bounds[group]
        else:
            continue

        if 'pf' in group:
            se = se.abs()

        store[group] = store[group].loc[(low < se) & (se < high)]
        print('{:.2f}% of sane {!s:<9} points inside div bounds'.format(np.sum(~store[group].isnull()) / pre * 100, group))


def stability_filter(data):
    for col in data.columns:
        splitted = re.compile('(?=.*)(.)(|ITG|ETG|TEM)_(GB|SI|cm)').split(col)
        if splitted[0] not in heat_vars + particle_vars + momentum_vars:
            print('skipping {!s}'.format(col))
            continue
        if splitted[2] == 'TEM':
            gam_filter = 'tem'
        elif splitted[2] == 'ITG':
            gam_filter = 'itg'
        elif splitted[2] == 'ETG':
            gam_filter = 'elec'
        elif splitted[0] in heat_vars and splitted[1] == 'e':
{'fraction_non_alphanumeric': 0.1714352174924401, 'fraction_numerical': 0.02209816236334031, 'mean_word_length': 2.586321934945788, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330539', 'n_tokens_mistral': 1743, 'n_tokens_neox': 1671, 'n_words': 368}