<commit_before>"""This module contains the "Viz" objects

These objects represent the backend of all the visualizations that
Superset can render.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import hashlib
import logging
import traceback
import uuid
import zlib

from collections import OrderedDict, defaultdict
from itertools import product
from datetime import datetime, timedelta

import pandas as pd
import numpy as np
from flask import request
from flask_babel import lazy_gettext as _
from markdown import markdown
import simplejson as json
from six import string_types, PY3
from dateutil import relativedelta as rdelta

from superset import app, utils, cache, get_manifest_file
from superset.utils import DTTM_ALIAS

config = app.config
stats_logger = config.get('STATS_LOGGER')


class BaseViz(object):

    """All visualizations derive this base class"""

    viz_type = None
    verbose_name = "Base Viz"
    credits = ""
    is_timeseries = False

    def __init__(self, datasource, form_data):
        if not datasource:
            raise Exception(_("Viz is missing a datasource"))
        self.datasource = datasource
        self.request = request
        self.viz_type = form_data.get("viz_type")
        self.form_data = form_data

        self.query = ""
        self.token = self.form_data.get(
            'token', 'token_' + uuid.uuid4().hex[:8])
        self.metrics = self.form_data.get('metrics') or []
        self.groupby = self.form_data.get('groupby') or []

        self.status = None
        self.error_message = None

    def get_df(self, query_obj=None):
        """Returns a pandas dataframe based on the query object"""
        if not query_obj:
            query_obj = self.query_obj()

        self.error_msg = ""
        self.results = None

        timestamp_format = None
        if self.datasource.type == 'table':
            dttm_col = self.datasource.get_col(query_obj['granularity'])
            if dttm_col:
                timestamp_format = dttm_col.python_date_format

        # The datasource here can be different backend but the interface is common
        self.results = self.datasource.query(query_obj)
        self.query = self.results.query
        self.status = self.results.status
        self.error_message = self.results.error_message

        df = self.results.df
        # Transform the timestamp we received from database to pandas supported
        # datetime format. If no python_date_format is specified, the pattern will
        # be considered as the default ISO date format
        # If the datetime format is unix, the parse will use the corresponding
        # parsing logic.
        if df is None or df.empty:
            self.status = utils.QueryStatus.FAILED
            if not self.error_message:
                self.error_message = "No data."
            return pd.DataFrame()
        else:
            if DTTM_ALIAS in df.columns:
                if timestamp_format in ("epoch_s", "epoch_ms"):
                    df[DTTM_ALIAS] = pd.to_datetime(df[DTTM_ALIAS], utc=False)
                else:
                    df[DTTM_ALIAS] = pd.to_datetime(
                        df[DTTM_ALIAS], utc=False, format=timestamp_format)
                if self.datasource.offset:
                    df[DTTM_ALIAS] += timedelta(hours=self.datasource.offset)
            df.replace([np.inf, -np.inf], np.nan)
            df = df.fillna(0)
        return df

    def get_extra_filters(self):
        extra_filters = self.form_data.get('extra_filters', [])
        return {f['col']: f['val'] for f in extra_filters}

    def query_obj(self):
        """Building a query object"""
        form_data = self.form_data
        gb = form_data.get("groupby") or []
        metrics = form_data.get("metrics") or []
        columns = form_data.get("columns") or []
        groupby = []
        for o in gb + columns:
            if o not in groupby:
                groupby.append(o)

        is_timeseries = self.is_timeseries
        if DTTM_ALIAS in groupby:
            groupby.remove(DTTM_ALIAS)
            is_timeseries = True

        # extra_filters are temporary/contextual filters that are external
        # to the slice definition. We use those for dynamic interactive
        # filters like the ones emitted by the "Filter Box" visualization
        extra_filters = self.get_extra_filters()
        granularity = (
            form_data.get("granularity") or form_data.get("granularity_sqla")
        )
        limit = int(form_data.get("limit") or 0)
        timeseries_limit_metric = form_data.get("timeseries_limit_metric")
        row_limit = int(
            form_data.get("row_limit") or config.get("ROW_LIMIT"))

        # __form and __to are special extra_filters that target time
        # boundaries. The rest of extra_filters are simple
        # [column_name in list_of_values]. `__` prefix is there to avoid
        # potential conflicts with column that would be named `from` or `to`
        since = (
            extra_filters.get('__from') or
            form_data.get("since")
        )

        from_dttm = utils.parse_human_datetime(since)

        until = extra_filters.get('__to') or form_data.get("until", "now")
        to_dttm = utils.parse_human_datetime(until)
        if from_dttm and to_dttm and from_dttm > to_dttm:
            raise Exception(_("From date cannot be larger than to date"))

        # extras are used to query elements specific to a datasource type
        # for instance the extra where clause that applies only to Tables
        extras = {
            'where': form_data.get("where", ''),
            'having': form_data.get("having", ''),
            'having_druid': form_data.get('having_filters') \<commit_msg>[hotfix] backward compatibility on date expressions (#3396)<commit_after>"""This module contains the "Viz" objects

These objects represent the backend of all the visualizations that
Superset can render.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import hashlib
import logging
import traceback
import uuid
import zlib

from collections import OrderedDict, defaultdict
from itertools import product
from datetime import datetime, timedelta

import pandas as pd
import numpy as np
from flask import request
from flask_babel import lazy_gettext as _
from markdown import markdown
import simplejson as json
from six import string_types, PY3
from dateutil import relativedelta as rdelta

from superset import app, utils, cache, get_manifest_file
from superset.utils import DTTM_ALIAS

config = app.config
stats_logger = config.get('STATS_LOGGER')


class BaseViz(object):

    """All visualizations derive this base class"""

    viz_type = None
    verbose_name = "Base Viz"
    credits = ""
    is_timeseries = False

    def __init__(self, datasource, form_data):
        if not datasource:
            raise Exception(_("Viz is missing a datasource"))
        self.datasource = datasource
        self.request = request
        self.viz_type = form_data.get("viz_type")
        self.form_data = form_data

        self.query = ""
        self.token = self.form_data.get(
            'token', 'token_' + uuid.uuid4().hex[:8])
        self.metrics = self.form_data.get('metrics') or []
        self.groupby = self.form_data.get('groupby') or []

        self.status = None
        self.error_message = None

    def get_df(self, query_obj=None):
        """Returns a pandas dataframe based on the query object"""
        if not query_obj:
            query_obj = self.query_obj()

        self.error_msg = ""
        self.results = None

        timestamp_format = None
        if self.datasource.type == 'table':
            dttm_col = self.datasource.get_col(query_obj['granularity'])
            if dttm_col:
                timestamp_format = dttm_col.python_date_format

        # The datasource here can be different backend but the interface is common
        self.results = self.datasource.query(query_obj)
        self.query = self.results.query
        self.status = self.results.status
        self.error_message = self.results.error_message

        df = self.results.df
        # Transform the timestamp we received from database to pandas supported
        # datetime format. If no python_date_format is specified, the pattern will
        # be considered as the default ISO date format
        # If the datetime format is unix, the parse will use the corresponding
        # parsing logic.
        if df is None or df.empty:
            self.status = utils.QueryStatus.FAILED
            if not self.error_message:
                self.error_message = "No data."
            return pd.DataFrame()
        else:
            if DTTM_ALIAS in df.columns:
                if timestamp_format in ("epoch_s", "epoch_ms"):
                    df[DTTM_ALIAS] = pd.to_datetime(df[DTTM_ALIAS], utc=False)
                else:
                    df[DTTM_ALIAS] = pd.to_datetime(
                        df[DTTM_ALIAS], utc=False, format=timestamp_format)
                if self.datasource.offset:
                    df[DTTM_ALIAS] += timedelta(hours=self.datasource.offset)
            df.replace([np.inf, -np.inf], np.nan)
            df = df.fillna(0)
        return df

    def get_extra_filters(self):
        extra_filters = self.form_data.get('extra_filters', [])
        return {f['col']: f['val'] for f in extra_filters}

    def query_obj(self):
        """Building a query object"""
        form_data = self.form_data
        gb = form_data.get("groupby") or []
        metrics = form_data.get("metrics") or []
        columns = form_data.get("columns") or []
        groupby = []
        for o in gb + columns:
            if o not in groupby:
                groupby.append(o)

        is_timeseries = self.is_timeseries
        if DTTM_ALIAS in groupby:
            groupby.remove(DTTM_ALIAS)
            is_timeseries = True

        # extra_filters are temporary/contextual filters that are external
        # to the slice definition. We use those for dynamic interactive
        # filters like the ones emitted by the "Filter Box" visualization
        extra_filters = self.get_extra_filters()
        granularity = (
            form_data.get("granularity") or form_data.get("granularity_sqla")
        )
        limit = int(form_data.get("limit") or 0)
        timeseries_limit_metric = form_data.get("timeseries_limit_metric")
        row_limit = int(
            form_data.get("row_limit") or config.get("ROW_LIMIT"))

        # __form and __to are special extra_filters that target time
        # boundaries. The rest of extra_filters are simple
        # [column_name in list_of_values]. `__` prefix is there to avoid
        # potential conflicts with column that would be named `from` or `to`
        since = (
            extra_filters.get('__from') or
            form_data.get("since")
        )

        # Backward compatibility hack
        since_words = since.split(' ')
        if (
                len(since_words) == 2 and
                since_words[1] in ['days', 'years', 'hours', 'day', 'year']):
            since += ' ago'

        from_dttm = utils.parse_human_datetime(since)

        until = extra_filters.get('__to') or form_data.get("until", "now")
        to_dttm = utils.parse_human_datetime(until)
        if from_dttm and to_dttm and from_dttm > to_dttm:
            raise Exception(_("From date cannot be larger than to date"))

        # extras are used to query elements specific to a datasource type
        # for instance the extra where clause that applies only to Tables
        extras = {
            'where': form_data.get("where", ''),
            'having': form_data.get("having", ''),
            'having_druid': form_data.get('having_filters') \
{'fraction_non_alphanumeric': 0.08453642660435617, 'fraction_numerical': 0.0013352249019444214, 'mean_word_length': 2.4259576901086337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6490050', 'n_tokens_mistral': 3493, 'n_tokens_neox': 3386, 'n_words': 1010}