<commit_before>import logging
from typing import Iterable, List, Tuple, Type

import pandas as pd
from pypika.queries import QueryBuilder

from fireant.database import Database
from fireant.dataset.fields import DataType, Field
from fireant.dataset.references import calculate_delta_percent
from fireant.dataset.totals import get_totals_marker_for_dtype
from fireant.utils import alias_selector, chunks
from .finders import find_field_in_modified_field, find_totals_dimensions
from .pandas_workaround import df_subtract
from ..dataset.modifiers import RollupValue

logger = logging.getLogger(__name__)

# Passing in an empty dictionary as format option to pandas' parse_dates causes errors to be ignored instead of
# being coerced into NaT.
PANDAS_TO_DATETIME_FORMAT = {}


def fetch_data(
    database: Database,
    queries: List[Type[QueryBuilder]],
    dimensions: Iterable[Field],
    share_dimensions: Iterable[Field] = (),
    reference_groups=(),
) -> Tuple[int, pd.DataFrame]:
    queries = [str(query) for query in queries]

    # Indicate which dimensions need to be parsed as date types
    # For this we create a dictionary with the dimension alias as key and PANDAS_TO_DATETIME_FORMAT as value
    pandas_parse_dates = {}
    for dimension in dimensions:
        unmodified_dimension = find_field_in_modified_field(dimension)
        if unmodified_dimension.data_type == DataType.date:
            pandas_parse_dates[alias_selector(unmodified_dimension.alias)] = PANDAS_TO_DATETIME_FORMAT

    results = database.fetch_dataframes(*queries, parse_dates=pandas_parse_dates)
    max_rows_returned = max([len(x) for x in results], default=0)
    logger.info('max_rows_returned', extra={'row_count': max_rows_returned, 'database': str(Database)})

    return max_rows_returned, reduce_result_set(results, reference_groups, dimensions, share_dimensions)


def reduce_result_set(
    results: Iterable[pd.DataFrame],
    reference_groups,
    dimensions: Iterable[Field],
    share_dimensions: Iterable[Field],
):
    """
    Reduces the result sets from individual queries into a single data frame. This effectively joins sets of references
    and concatenates the sets of totals.<commit_msg>Fix database stringifying in log line<commit_after>import logging
from typing import Iterable, List, Tuple, Type

import pandas as pd
from pypika.queries import QueryBuilder

from fireant.database import Database
from fireant.dataset.fields import DataType, Field
from fireant.dataset.references import calculate_delta_percent
from fireant.dataset.totals import get_totals_marker_for_dtype
from fireant.utils import alias_selector, chunks
from .finders import find_field_in_modified_field, find_totals_dimensions
from .pandas_workaround import df_subtract
from ..dataset.modifiers import RollupValue

logger = logging.getLogger(__name__)

# Passing in an empty dictionary as format option to pandas' parse_dates causes errors to be ignored instead of
# being coerced into NaT.
PANDAS_TO_DATETIME_FORMAT = {}


def fetch_data(
    database: Database,
    queries: List[Type[QueryBuilder]],
    dimensions: Iterable[Field],
    share_dimensions: Iterable[Field] = (),
    reference_groups=(),
) -> Tuple[int, pd.DataFrame]:
    queries = [str(query) for query in queries]

    # Indicate which dimensions need to be parsed as date types
    # For this we create a dictionary with the dimension alias as key and PANDAS_TO_DATETIME_FORMAT as value
    pandas_parse_dates = {}
    for dimension in dimensions:
        unmodified_dimension = find_field_in_modified_field(dimension)
        if unmodified_dimension.data_type == DataType.date:
            pandas_parse_dates[alias_selector(unmodified_dimension.alias)] = PANDAS_TO_DATETIME_FORMAT

    results = database.fetch_dataframes(*queries, parse_dates=pandas_parse_dates)
    max_rows_returned = max([len(x) for x in results], default=0)
    logger.info('max_rows_returned', extra={'row_count': max_rows_returned, 'database': str(database)})

    return max_rows_returned, reduce_result_set(results, reference_groups, dimensions, share_dimensions)


def reduce_result_set(
    results: Iterable[pd.DataFrame],
    reference_groups,
    dimensions: Iterable[Field],
    share_dimensions: Iterable[Field],
):
    """
    Reduces the result sets from individual queries into a single data frame. This effectively joins sets of references
    and concatenates the sets of totals.
{'fraction_non_alphanumeric': 0.06913691821057388, 'fraction_numerical': 0.00045187528242205153, 'mean_word_length': 5.148611111111111, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '94880', 'n_tokens_mistral': 1286, 'n_tokens_neox': 1271, 'n_words': 380}