<commit_before>import os
import sys

import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
import pandas
import openpyxl

from wqio import utils
from wqio import Parameter

from . import dataAccess, info

from statsmodels.tools.decorators import (
    resettable_cache, cache_readonly, cache_writable
)



def filterlocation(location, count=5, column='bmp'):
    location.filtered_data = (
        location.filtered_data
        .groupby(level=column)
        .filter(lambda g: g.count() >= count)
    )

    location.include = (
        location.filtered_data
                .index
                .get_level_values(column)
                .unique()
                .shape[0]
    ) >= count


def getPFCs(db):
    # get BMP Name of pervious friction course (PFC) BMPs
    bmpnamecol = 'BMPNAME'
    bmptable = 'BMP INFO S02'
    bmptypecol = 'TBMPT 2009'
    query = """
    select [{0}]
    FROM [{1}]
    WHERE [{2}] = 'PF';
    """.format(bmpnamecol, bmptable, bmptypecol)

    with db.connect() as cnn:
        pfc_names = pandas.read_sql(query, cnn)[bmpnamecol].tolist()

    return pfc_names


def _pick_best_station(dataframe):
    def best_col(row, mainstation, backupstation, valcol):
        if pandas.isnull(row[(mainstation, valcol)]):
            return row[(backupstation, valcol)]
        else:
            return row[(mainstation, valcol)]

    xtab = dataframe.unstack(level='station')
    xtab.columns = xtab.columns.swaplevel(0, 1)
    xtab[('final_outflow', 'res')] = xtab.apply(
        best_col, axis=1, args=('outflow', 'subsurface', 'res')
    )
    xtab[('final_outflow', 'qual')] = xtab.apply(
        best_col, axis=1, args=('outflow', 'subsurface', 'qual')
    )
    xtab[('final_inflow', 'qual')] = xtab.apply(
        best_col, axis=1, args=('inflow', 'reference outflow', 'qual')
    )
    xtab[('final_inflow', 'res')] = xtab.apply(
        best_col, axis=1, args=('inflow', 'reference outflow', 'res')
    )

    data = (
        xtab.select(lambda c: 'final_' in c[0], axis=1)<commit_msg>handle cases when the backup station never exists in a group<commit_after>import os
import sys

import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
import pandas
import openpyxl

from wqio import utils
from wqio import Parameter

from . import dataAccess, info

from statsmodels.tools.decorators import (
    resettable_cache, cache_readonly, cache_writable
)



def filterlocation(location, count=5, column='bmp'):
    location.filtered_data = (
        location.filtered_data
        .groupby(level=column)
        .filter(lambda g: g.count() >= count)
    )

    location.include = (
        location.filtered_data
                .index
                .get_level_values(column)
                .unique()
                .shape[0]
    ) >= count


def getPFCs(db):
    # get BMP Name of pervious friction course (PFC) BMPs
    bmpnamecol = 'BMPNAME'
    bmptable = 'BMP INFO S02'
    bmptypecol = 'TBMPT 2009'
    query = """
    select [{0}]
    FROM [{1}]
    WHERE [{2}] = 'PF';
    """.format(bmpnamecol, bmptable, bmptypecol)

    with db.connect() as cnn:
        pfc_names = pandas.read_sql(query, cnn)[bmpnamecol].tolist()

    return pfc_names


def _pick_best_station(dataframe):
    def best_col(row, mainstation, backupstation, valcol):
        try:
            if pandas.isnull(row[(mainstation, valcol)]):
                return row[(backupstation, valcol)]
            else:
                return row[(mainstation, valcol)]
        except KeyError:
            return np.nan

    xtab = dataframe.unstack(level='station')
    xtab.columns = xtab.columns.swaplevel(0, 1)
    xtab[('final_outflow', 'res')] = xtab.apply(
        best_col, axis=1, args=('outflow', 'subsurface', 'res')
    )
    xtab[('final_outflow', 'qual')] = xtab.apply(
        best_col, axis=1, args=('outflow', 'subsurface', 'qual')
    )
    xtab[('final_inflow', 'qual')] = xtab.apply(
        best_col, axis=1, args=('inflow', 'reference outflow', 'qual')
    )
    xtab[('final_inflow', 'res')] = xtab.apply(
        best_col, axis=1, args=('inflow', 'reference outflow', 'res')
    )

    data = (
        xtab.select(lambda c: 'final_' in c[0], axis=1)
{'fraction_non_alphanumeric': 0.13023147065700258, 'fraction_numerical': 0.008884732288987608, 'mean_word_length': 2.795918367346939, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3575436', 'n_tokens_mistral': 1497, 'n_tokens_neox': 1423, 'n_words': 336}