<commit_before>import numbers
from collections import OrderedDict

from django.utils.decorators import method_decorator
from django.utils.translation import ugettext

from dimagi.utils.decorators.memoized import memoized

from corehq.apps.reports.api import ReportDataSource
from corehq.apps.userreports.decorators import catch_and_raise_exceptions
from corehq.apps.userreports.exceptions import InvalidQueryColumn
from corehq.apps.userreports.models import DataSourceConfiguration, get_datasource_config
from corehq.apps.userreports.reports.sorting import ASCENDING
from corehq.apps.userreports.reports.util import get_expanded_columns
from corehq.apps.userreports.util import get_table_name

from corehq.apps.es.es_query import HQESQuery


class ConfigurableReportEsDataSource(ReportDataSource):

    def __init__(self, domain, config_or_config_id, filters, aggregation_columns, columns, order_by):
        self.lang = None
        self.domain = domain
        if isinstance(config_or_config_id, DataSourceConfiguration):
            self._config = config_or_config_id
            self._config_id = self._config._id
        else:
            assert isinstance(config_or_config_id, basestring)
            self._config = None
            self._config_id = config_or_config_id

        self._filters = {f.slug: f for f in filters}
        self._filter_values = {}
        self._deferred_filters = {}
        self._order_by = order_by
        self._aggregation_columns = aggregation_columns
        self._column_configs = OrderedDict()
        for column in columns:
            # should be caught in validation prior to reaching this
            assert column.column_id not in self._column_configs, \
                'Report {} in domain {} has more than one {} column defined!'.format(
                    self._config_id, self.domain, column.column_id,
                )
            self._column_configs[column.column_id] = column

    @property
    def aggregation_columns(self):
        # TODO add deferred filters to support mobile ucr
        return self._aggregation_columns

    @property
    def config(self):
        if self._config is None:
            self._config, _ = get_datasource_config(self._config_id, self.domain)
        return self._config

    @property
    def column_configs(self):
        return self._column_configs.values()

    @property
    def table_name(self):
        # TODO make this the same function as the adapter
        return get_table_name(self.domain, self.config.table_id).lower()

    @property
    def filters(self):
        return filter(None, [f.to_es_filter() for f in self._filters.values()])
    #
    # def set_filter_values(self, filter_values):
    #     for filter_slug, value in filter_values.items():
    #         self._filter_values[filter_slug] = self._filters[filter_slug].create_filter_value(value)

    def defer_filters(self, filter_slugs):
        self._deferred_filters.update({
            filter_slug: self._filters[filter_slug] for filter_slug in filter_slugs})

    # def set_order_by(self, columns):
    #     self._order_by = columns
    #
    # @property
    # def filter_values(self):
    #     return {k: v for fv in self._filter_values.values() for k, v in fv.to_sql_values().items()}
    #
    # @property
    # def group_by(self):
    #     # ask each column for its group_by contribution and combine to a single list
    #     return [
    #         group_by for col_id in self.aggregation_columns
    #         for group_by in self._get_db_column_ids(col_id)
    #     ]
    #
    # @property
    # def order_by(self):
    #     # allow throwing exception if the report explicitly sorts on an unsortable column type
    #     if self._order_by:
    #         return [
    #             OrderBy(order_by, is_ascending=(order == ASCENDING))
    #             for sort_column_id, order in self._order_by
    #             for order_by in self._get_db_column_ids(sort_column_id)
    #         ]
    #     elif self.column_configs:
    #         try:
    #             return [
    #                 OrderBy(order_by, is_ascending=True)
    #                 for order_by in self._get_db_column_ids(self.column_configs[0].column_id)
    #             ]
    #         except InvalidQueryColumn:
    #             pass
    #     return []
    #
    # @property
    # def columns(self):
    #     db_columns = [c for sql_conf in self.sql_column_configs for c in sql_conf.columns]
    #     fields = {c.slug for c in db_columns}
    #
    #     return db_columns + [
    #         DatabaseColumn('', SimpleColumn(deferred_filter.field))
    #         for deferred_filter in self._deferred_filters.values()
    #         if deferred_filter.field not in fields]
    #
    # @property
    # def sql_column_configs(self):
    #     return [col.get_sql_column_config(self.config, self.lang) for col in self.column_configs]
    #
    # @property
    # def column_warnings(self):
    #     return [w for es_conf in self.es_column_configs for w in es_conf.warnings]

    @memoized
    @method_decorator(catch_and_raise_exceptions)
    def get_data(self, start=None, limit=None):
        hits = self._get_query(start, limit).hits
        ret = []

        for row in hits:
            r = {}
            for report_column in self.column_configs:
                r[report_column.column_id] = row[report_column.field]
            ret.append(r)

        for report_column in self.column_configs:
            report_column.format_data(ret)
<commit_msg>TEMPORARY: Use sql column configs so I can see things in the UI<commit_after>import numbers
from collections import OrderedDict

from django.utils.decorators import method_decorator
from django.utils.translation import ugettext

from dimagi.utils.decorators.memoized import memoized

from corehq.apps.reports.api import ReportDataSource
from corehq.apps.userreports.decorators import catch_and_raise_exceptions
from corehq.apps.userreports.exceptions import InvalidQueryColumn
from corehq.apps.userreports.models import DataSourceConfiguration, get_datasource_config
from corehq.apps.userreports.reports.sorting import ASCENDING
from corehq.apps.userreports.reports.util import get_expanded_columns
from corehq.apps.userreports.util import get_table_name

from corehq.apps.es.es_query import HQESQuery


class ConfigurableReportEsDataSource(ReportDataSource):

    def __init__(self, domain, config_or_config_id, filters, aggregation_columns, columns, order_by):
        self.lang = None
        self.domain = domain
        if isinstance(config_or_config_id, DataSourceConfiguration):
            self._config = config_or_config_id
            self._config_id = self._config._id
        else:
            assert isinstance(config_or_config_id, basestring)
            self._config = None
            self._config_id = config_or_config_id

        self._filters = {f.slug: f for f in filters}
        self._filter_values = {}
        self._deferred_filters = {}
        self._order_by = order_by
        self._aggregation_columns = aggregation_columns
        self._column_configs = OrderedDict()
        for column in columns:
            # should be caught in validation prior to reaching this
            assert column.column_id not in self._column_configs, \
                'Report {} in domain {} has more than one {} column defined!'.format(
                    self._config_id, self.domain, column.column_id,
                )
            self._column_configs[column.column_id] = column

    @property
    def aggregation_columns(self):
        # TODO add deferred filters to support mobile ucr
        return self._aggregation_columns

    @property
    def config(self):
        if self._config is None:
            self._config, _ = get_datasource_config(self._config_id, self.domain)
        return self._config

    @property
    def column_configs(self):
        return self._column_configs.values()

    @property
    def table_name(self):
        # TODO make this the same function as the adapter
        return get_table_name(self.domain, self.config.table_id).lower()

    @property
    def filters(self):
        return filter(None, [f.to_es_filter() for f in self._filter_values.values()])

    def set_filter_values(self, filter_values):
        for filter_slug, value in filter_values.items():
            self._filter_values[filter_slug] = self._filters[filter_slug].create_filter_value(value)

    def defer_filters(self, filter_slugs):
        self._deferred_filters.update({
            filter_slug: self._filters[filter_slug] for filter_slug in filter_slugs})

    # def set_order_by(self, columns):
    #     self._order_by = columns
    #
    # @property
    # def filter_values(self):
    #     return {k: v for fv in self._filter_values.values() for k, v in fv.to_sql_values().items()}
    #
    # @property
    # def group_by(self):
    #     # ask each column for its group_by contribution and combine to a single list
    #     return [
    #         group_by for col_id in self.aggregation_columns
    #         for group_by in self._get_db_column_ids(col_id)
    #     ]
    #
    # @property
    # def order_by(self):
    #     # allow throwing exception if the report explicitly sorts on an unsortable column type
    #     if self._order_by:
    #         return [
    #             OrderBy(order_by, is_ascending=(order == ASCENDING))
    #             for sort_column_id, order in self._order_by
    #             for order_by in self._get_db_column_ids(sort_column_id)
    #         ]
    #     elif self.column_configs:
    #         try:
    #             return [
    #                 OrderBy(order_by, is_ascending=True)
    #                 for order_by in self._get_db_column_ids(self.column_configs[0].column_id)
    #             ]
    #         except InvalidQueryColumn:
    #             pass
    #     return []

    @property
    def columns(self):
        db_columns = [c for sql_conf in self.sql_column_configs for c in sql_conf.columns]
        return db_columns

    @property
    def sql_column_configs(self):
        return [col.get_sql_column_config(self.config, self.lang) for col in self.column_configs]

    @property
    def column_warnings(self):
        return [w for es_conf in self.sql_column_configs for w in es_conf.warnings]

    @memoized
    @method_decorator(catch_and_raise_exceptions)
    def get_data(self, start=None, limit=None):
        hits = self._get_query(start, limit).hits
        ret = []

        for row in hits:
            r = {}
            for report_column in self.column_configs:
                r[report_column.column_id] = row[report_column.field]
            ret.append(r)

        for report_column in self.column_configs:
            report_column.format_data(ret)

{'fraction_non_alphanumeric': 0.07710150306179255, 'fraction_numerical': 0.00018556318426424197, 'mean_word_length': 2.559775429326288, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '356526', 'n_tokens_mistral': 3196, 'n_tokens_neox': 3135, 'n_words': 745}