<commit_before>"""
The :mod:`hillmaker.bydatetime` module includes functions for computing occupancy,
arrival, and departure statistics by time bin of day and date.
"""

# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pandas as pd
from pandas import DataFrame
from pandas import Series
from pandas import Timestamp
from datetime import datetime
from datetime import timedelta
from pandas.tseries.offsets import Minute
import itertools
from timeit import default_timer as timer


from hillmaker.hmlib import *


def make_bydatetime(stops_df, infield, outfield,
                    start_analysis, end_analysis, catfield=None,
                    bin_size_minutes=60,
                    cat_to_exclude=None,
                    totals=1,
                    edge_bins=1,
                    verbose=0):
    """
    Create bydatetime table based on user inputs.

    This is the table from which summary statistics can be computed.

    Parameters
    ----------
    stops_df: DataFrame
        Stop data

    infield: string
        Name of column in stops_df to use as arrival datetime

    outfield: string
        Name of column in stops_df to use as departure datetime

    start_analysis: datetime
        Start date for the analysis

    end_analysis: datetime
        End date for the analysis

    catfield : string or List of strings, optional
        Column name(s) corresponding to the categories. If none is specified, then only overall occupancy is analyzed.

    bin_size_minutes: int, default 60
        Bin size in minutes. Should divide evenly into 1440.

    cat_to_exclude: list of strings, default None
        Categories to ignore

    edge_bins: int, default 1
        Occupancy contribution method for arrival and departure bins. 1=fractional, 2=whole bin

    totals: int, default 1
        0=no totals, 1=totals by datetime, 2=totals bydatetime as well as totals for each field in the
        catfields (only relevant for > 1 category field)

    verbose : int, default 0
        The verbosity level. The default, zero, means silent mode.

    Returns
    -------
    Dict of DataFrames
       Occupancy, arrivals, departures by category by datetime bin

    Examples
    --------
    bydt_dfs = make_bydatetime(stops_df, 'InTime', 'OutTime',
    ...                        datetime(2014, 3, 1), datetime(2014, 6, 30), 'PatientType', 60)

    bydt_dfs = make_bydatetime(stops_df, 'InTime', 'OutTime',
    ...           datetime(2014, 3, 1), datetime(2014, 6, 30), ['PatientType','Severity'], 60, totals=2)


    TODO
    ----


     Notes
    -----


    References
    ----------


    See Also
    --------
    """

    start_analysis_dt = pd.Timestamp(start_analysis)
    end_analysis_dt = pd.Timestamp(end_analysis)

    # Compute min and max of in and out times
    min_intime = stops_df[infield].min()
    max_intime = stops_df[infield].max()
    min_outtime = stops_df[outfield].min()
    max_outtime = stops_df[outfield].max()

    if verbose:
        print("min of intime: {}".format(min_intime))
        print("max of outtime: {}".format(max_outtime))
        print("max of intime: {}".format(max_intime))
        print("min of outtime: {}".format(min_outtime))

    # TODO - Add warnings here related to min and maxes out of whack with analysis range

    analysis_range = [start_analysis_dt, end_analysis_dt]

    rng_bydt = Series(pd.date_range(start_analysis_dt, end_analysis_dt, freq=Minute(bin_size_minutes)))

    # Handle cases of no catfield, a single fieldname, or a list of fields
    # If no category, add a temporary dummy column populated with a totals str

    CONST_FAKE_CATFIELDNAME = '__FakeCatForTotals__'
    total_str = 'total'

    bTotalsDone = False
    if catfield is not None:
        if isinstance(catfield, str):
            catfield = [catfield]
    else:
        totlist = [total_str]*len(stops_df)
        totseries = Series(totlist,dtype=str,name=[CONST_FAKE_CATFIELDNAME])
        totfield_df = DataFrame({CONST_FAKE_CATFIELDNAME: totseries})
        stops_df = pd.concat([stops_df, totfield_df],axis=1)
        catfield = [CONST_FAKE_CATFIELDNAME]
        bTotalsDone = True

    # Get the unique category values and exclude any specified to exclude
    categories = []
    if cat_to_exclude is not None:
        for i in range(len(catfield)):
            categories.append(tuple([c for c in stops_df[catfield[i]].unique() if c not in cat_to_exclude[i]]))
    else:
        for i in range(len(catfield)):
            categories.append(tuple([c for c in stops_df[catfield[i]].unique()]))

    for i in range(len(catfield)):
        stops_df = stops_df[stops_df[catfield[i]].isin(categories[i])]

    # Now we'll build up the seeded by date table a category at a time.
    # Along the way we'll initialize all the measures to 0.

    # The following doesn't feel very Pythonic, but just trying to get it working for now
    len_bydt = len(rng_bydt)
    all_cat_df = []<commit_msg>added comment regarding multicat input<commit_after>"""
The :mod:`hillmaker.bydatetime` module includes functions for computing occupancy,
arrival, and departure statistics by time bin of day and date.
"""

# Copyright 2015 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pandas as pd
from pandas import DataFrame
from pandas import Series
from pandas import Timestamp
from datetime import datetime
from datetime import timedelta
from pandas.tseries.offsets import Minute
import itertools
from timeit import default_timer as timer


from hillmaker.hmlib import *


def make_bydatetime(stops_df, infield, outfield,
                    start_analysis, end_analysis, catfield=None,
                    bin_size_minutes=60,
                    cat_to_exclude=None,
                    totals=1,
                    edge_bins=1,
                    verbose=0):
    """
    Create bydatetime table based on user inputs.

    This is the table from which summary statistics can be computed.

    Parameters
    ----------
    stops_df: DataFrame
        Stop data

    infield: string
        Name of column in stops_df to use as arrival datetime

    outfield: string
        Name of column in stops_df to use as departure datetime

    start_analysis: datetime
        Start date for the analysis

    end_analysis: datetime
        End date for the analysis

    catfield : string or List of strings, optional
        Column name(s) corresponding to the categories. If none is specified, then only overall occupancy is analyzed.

    bin_size_minutes: int, default 60
        Bin size in minutes. Should divide evenly into 1440.

    cat_to_exclude: list of strings, default None
        Categories to ignore

    edge_bins: int, default 1
        Occupancy contribution method for arrival and departure bins. 1=fractional, 2=whole bin

    totals: int, default 1
        0=no totals, 1=totals by datetime, 2=totals bydatetime as well as totals for each field in the
        catfields (only relevant for > 1 category field)

    verbose : int, default 0
        The verbosity level. The default, zero, means silent mode.

    Returns
    -------
    Dict of DataFrames
       Occupancy, arrivals, departures by category by datetime bin

    Examples
    --------
    bydt_dfs = make_bydatetime(stops_df, 'InTime', 'OutTime',
    ...                        datetime(2014, 3, 1), datetime(2014, 6, 30), 'PatientType', 60)

    bydt_dfs = make_bydatetime(stops_df, 'InTime', 'OutTime',
    ...           datetime(2014, 3, 1), datetime(2014, 6, 30), ['PatientType','Severity'], 60, totals=2)


    TODO
    ----


     Notes
    -----


    References
    ----------


    See Also
    --------
    """

    start_analysis_dt = pd.Timestamp(start_analysis)
    end_analysis_dt = pd.Timestamp(end_analysis)

    # Compute min and max of in and out times
    min_intime = stops_df[infield].min()
    max_intime = stops_df[infield].max()
    min_outtime = stops_df[outfield].min()
    max_outtime = stops_df[outfield].max()

    if verbose:
        print("min of intime: {}".format(min_intime))
        print("max of outtime: {}".format(max_outtime))
        print("max of intime: {}".format(max_intime))
        print("min of outtime: {}".format(min_outtime))

    # TODO - Add warnings here related to min and maxes out of whack with analysis range

    analysis_range = [start_analysis_dt, end_analysis_dt]

    rng_bydt = Series(pd.date_range(start_analysis_dt, end_analysis_dt, freq=Minute(bin_size_minutes)))

    # Handle cases of no catfield, a single fieldname, or a list of fields
    # If no category, add a temporary dummy column populated with a totals str

    CONST_FAKE_CATFIELDNAME = '__FakeCatForTotals__'
    total_str = 'total'

    bTotalsDone = False
    if catfield is not None:
        # If it's a string, it's a single cat field --> convert to list
        if isinstance(catfield, str):
            catfield = [catfield]
    else:
        totlist = [total_str]*len(stops_df)
        totseries = Series(totlist,dtype=str,name=[CONST_FAKE_CATFIELDNAME])
        totfield_df = DataFrame({CONST_FAKE_CATFIELDNAME: totseries})
        stops_df = pd.concat([stops_df, totfield_df],axis=1)
        catfield = [CONST_FAKE_CATFIELDNAME]
        bTotalsDone = True

    # Get the unique category values and exclude any specified to exclude
    categories = []
    if cat_to_exclude is not None:
        for i in range(len(catfield)):
            categories.append(tuple([c for c in stops_df[catfield[i]].unique() if c not in cat_to_exclude[i]]))
    else:
        for i in range(len(catfield)):
            categories.append(tuple([c for c in stops_df[catfield[i]].unique()]))

    for i in range(len(catfield)):
        stops_df = stops_df[stops_df[catfield[i]].isin(categories[i])]

    # Now we'll build up the seeded by date table a category at a time.
    # Along the way we'll initialize all the measures to 0.

    # The following doesn't feel very Pythonic, but just trying to get it working for now
    len_bydt = len(rng_bydt)
    all_cat_df = []
{'fraction_non_alphanumeric': 0.08462304280930401, 'fraction_numerical': 0.011041723232871753, 'mean_word_length': 2.827502597852442, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '735577', 'n_tokens_mistral': 3364, 'n_tokens_neox': 3120, 'n_words': 1142}