<commit_before># encoding: utf-8

from __future__ import division

import itertools
from collections import OrderedDict

import numpy as np
import pandas as pd


def _drop_zeros(df):
    """
    Drop zeros from a DataFrame, returning an iterator over the columns
    in the DataFrame.

    Yields tuples of (column name, non-zero column values, non-zero indexes).

    Parameters
    ----------
    df : pandas.DataFrame

    """
    for col_idx, col in df.iteritems():
        nz = col.nonzero()[0]
        yield (col_idx, col.values[nz], nz)


class _FrequencyAndConstraints(object):
    """
    Wrap frequency tables and constraints for both household and
    person classes for easy iteration over all of them.

    Also tracks the locations of non-zero elements in each column
    of the frequency tables. If including person classes, both
    `person_freq` and `person_constraints` are required.

    Parameters
    ----------
    household_freq : pandas.DataFrame
        Frequency table for household attributes. Columns should be
        a MultiIndex matching the index of `household_constraints` and
        index should be household IDs matching the index of
        `person_freq`.
    household_constraints : pandas.Series<commit_msg>massive speed up for ipu<commit_after># encoding: utf-8

from __future__ import division

import itertools
from collections import OrderedDict

import numpy as np
import pandas as pd


def _drop_zeros(df):
    """
    Drop zeros from a DataFrame, returning an iterator over the columns
    in the DataFrame.

    Yields tuples of (column name, non-zero column values, non-zero indexes).

    Parameters
    ----------
    df : pandas.DataFrame

    """
    def for_each_col(col):
        nz = col.nonzero()[0]
        return col[nz], nz

    for (col_idx, (col, nz)) in df.apply(for_each_col, axis=0, raw=True).iteritems():
        yield (col_idx, col, nz)


class _FrequencyAndConstraints(object):
    """
    Wrap frequency tables and constraints for both household and
    person classes for easy iteration over all of them.

    Also tracks the locations of non-zero elements in each column
    of the frequency tables. If including person classes, both
    `person_freq` and `person_constraints` are required.

    Parameters
    ----------
    household_freq : pandas.DataFrame
        Frequency table for household attributes. Columns should be
        a MultiIndex matching the index of `household_constraints` and
        index should be household IDs matching the index of
        `person_freq`.
    household_constraints : pandas.Series
{'fraction_non_alphanumeric': 0.07443116081758581, 'fraction_numerical': 0.0019282684149633628, 'mean_word_length': 2.93030303030303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3351334', 'n_tokens_mistral': 716, 'n_tokens_neox': 682, 'n_words': 283}