<commit_before>#!/usr/bin/env python

# Copyright (c) 2014 Cloudera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''This module will run random queries against existing databases and compare the
   results.

'''
from decimal import Decimal
from itertools import izip
from logging import getLogger
from math import isinf, isnan
from os import getenv, symlink, unlink
from os.path import join as join_path
from random import choice
from string import ascii_lowercase, digits
from subprocess import call
from threading import current_thread, Thread
from tempfile import gettempdir
from time import time

from tests.comparison.types import BigInt
from tests.comparison.db_connector import (
    DbConnection,
    DbConnector,
    IMPALA,
    MYSQL,
    ORACLE,
    POSTGRESQL)
from tests.comparison.query_generator import QueryGenerator
from tests.comparison.model_translator import SqlWriter

LOG = getLogger(__name__)

class QueryResultComparator(object):
  '''Used for comparing the results of a Query across two databases'''

  # Used when comparing FLOAT values
  EPSILON = 0.1

  # The DECIMAL values will be rounded before comparison
  DECIMAL_PLACES = 2

  def __init__(self, ref_connection, test_connection):
    '''test/ref_connection arguments should be an instance of DbConnection'''
    ref_cursor = ref_connection.create_cursor()
    test_cursor = test_connection.create_cursor()

    self.ref_connection = ref_connection
    self.ref_sql_writer = SqlWriter.create(dialect=ref_connection.db_type)
    self.test_connection = test_connection
    self.test_sql_writer = SqlWriter.create(dialect=test_connection.db_type)

    self.query_executor = QueryExecutor(
        [ref_cursor, test_cursor],
        [self.ref_sql_writer, self.test_sql_writer],
        query_timeout_seconds=(3 * 60))

  @property
  def ref_db_type(self):
    return self.ref_connection.db_type

  def compare_query_results(self, query):
    '''Execute the query, compare the data, and return a ComparisonResult, which
       summarizes the outcome.
    '''
    comparison_result = ComparisonResult(query, self.ref_db_type)
    (ref_sql, ref_exception, ref_data_set), (test_sql, test_exception, test_data_set) = \
        self.query_executor.fetch_query_results(query)

    comparison_result.ref_sql = ref_sql
    comparison_result.test_sql = test_sql

    if ref_exception:
      comparison_result.exception = ref_exception
      error_message = str(ref_exception)
      if 'Year is out of valid range: 1400..10000' in error_message:
        # This comes from Postgresql. Overflow errors will be ignored.
        comparison_result.exception = TypeOverflow(error_message)
      LOG.debug('%s encountered an error running query: %s',
          self.ref_connection.db_type, ref_exception, exc_info=True)
      return comparison_result

    if test_exception:
      # "known errors" will be ignored
      error_message = str(test_exception)
      known_error = None
      if 'Expressions in the ORDER BY clause must not be constant' in error_message \
          or 'Expressions in the PARTITION BY clause must not be consta' in error_message:
        # It's too much work to avoid this bug. Just ignore it if it comes up.
        known_error = KnownError('https://issues.cloudera.org/browse/IMPALA-1354')
      if known_error:
        comparison_result.exception = known_error
      else:
        comparison_result.exception = test_exception
        LOG.debug('%s encountered an error running query: %s',
            self.test_connection.db_type, test_exception, exc_info=True)
      return comparison_result

    comparison_result.ref_row_count = len(ref_data_set)
    comparison_result.test_row_count = len(test_data_set)
    comparison_result.query_resulted_in_data = (comparison_result.test_row_count > 0
        or comparison_result.ref_row_count > 0)
    if comparison_result.ref_row_count != comparison_result.test_row_count:
      return comparison_result

    # Standardize data (round FLOATs) in each column, and sort the data set<commit_msg>QGen: Add a few more known issues<commit_after>#!/usr/bin/env python

# Copyright (c) 2014 Cloudera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''This module will run random queries against existing databases and compare the
   results.

'''
from decimal import Decimal
from itertools import izip
from logging import getLogger
from math import isinf, isnan
from os import getenv, symlink, unlink
from os.path import join as join_path
from random import choice
from string import ascii_lowercase, digits
from subprocess import call
from threading import current_thread, Thread
from tempfile import gettempdir
from time import time

from tests.comparison.types import BigInt
from tests.comparison.db_connector import (
    DbConnection,
    DbConnector,
    IMPALA,
    MYSQL,
    ORACLE,
    POSTGRESQL)
from tests.comparison.query_generator import QueryGenerator
from tests.comparison.model_translator import SqlWriter

LOG = getLogger(__name__)

class QueryResultComparator(object):
  '''Used for comparing the results of a Query across two databases'''

  # Used when comparing FLOAT values
  EPSILON = 0.1

  # The DECIMAL values will be rounded before comparison
  DECIMAL_PLACES = 2

  def __init__(self, ref_connection, test_connection):
    '''test/ref_connection arguments should be an instance of DbConnection'''
    ref_cursor = ref_connection.create_cursor()
    test_cursor = test_connection.create_cursor()

    self.ref_connection = ref_connection
    self.ref_sql_writer = SqlWriter.create(dialect=ref_connection.db_type)
    self.test_connection = test_connection
    self.test_sql_writer = SqlWriter.create(dialect=test_connection.db_type)

    self.query_executor = QueryExecutor(
        [ref_cursor, test_cursor],
        [self.ref_sql_writer, self.test_sql_writer],
        query_timeout_seconds=(3 * 60))

  @property
  def ref_db_type(self):
    return self.ref_connection.db_type

  def compare_query_results(self, query):
    '''Execute the query, compare the data, and return a ComparisonResult, which
       summarizes the outcome.
    '''
    comparison_result = ComparisonResult(query, self.ref_db_type)
    (ref_sql, ref_exception, ref_data_set), (test_sql, test_exception, test_data_set) = \
        self.query_executor.fetch_query_results(query)

    comparison_result.ref_sql = ref_sql
    comparison_result.test_sql = test_sql

    if ref_exception:
      comparison_result.exception = ref_exception
      error_message = str(ref_exception)
      if 'Year is out of valid range: 1400..10000' in error_message:
        # This comes from Postgresql. Overflow errors will be ignored.
        comparison_result.exception = TypeOverflow(error_message)
      LOG.debug('%s encountered an error running query: %s',
          self.ref_connection.db_type, ref_exception, exc_info=True)
      return comparison_result

    if test_exception:
      # "known errors" will be ignored
      error_message = str(test_exception)
      known_error = None
      if 'Expressions in the ORDER BY clause must not be constant' in error_message \
          or 'Expressions in the PARTITION BY clause must not be consta' in error_message:
        # It's too much work to avoid this bug. Just ignore it if it comes up.
        known_error = KnownError('https://issues.cloudera.org/browse/IMPALA-1354')
      elif 'GROUP BY expression must not contain aggregate functions' in error_message \
          or 'select list expression not produced by aggregation output' in error_message:
        known_error = KnownError('https://issues.cloudera.org/browse/IMPALA-1423')
      elif ('max(' in error_message or 'min(' in error_message) \
          and 'only supported with an UNBOUNDED PRECEDING start bound' in error_message:
        # This analytic isn't supported and ignoring this here is much easier than not
        # generating the query...
        known_error = KnownError('MAX UNBOUNDED PRECISION')
      elif 'IN and/or EXISTS subquery predicates are not supported in binary predicates' \
          in error_message:
        known_error = KnownError('https://issues.cloudera.org/browse/IMPALA-1418')
      elif 'Unsupported predicate with subquery' in error_message:
        known_error = KnownError('https://issues.cloudera.org/browse/IMPALA-1950')
      if known_error:
        comparison_result.exception = known_error
      else:
        comparison_result.exception = test_exception
        LOG.debug('%s encountered an error running query: %s',
            self.test_connection.db_type, test_exception, exc_info=True)
      return comparison_result

    comparison_result.ref_row_count = len(ref_data_set)
    comparison_result.test_row_count = len(test_data_set)
    comparison_result.query_resulted_in_data = (comparison_result.test_row_count > 0
        or comparison_result.ref_row_count > 0)
    if comparison_result.ref_row_count != comparison_result.test_row_count:
      return comparison_result

    # Standardize data (round FLOATs) in each column, and sort the data set
{'fraction_non_alphanumeric': 0.06797658033144785, 'fraction_numerical': 0.006946511858688101, 'mean_word_length': 3.8874878758486906, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 5, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '963699', 'n_tokens_mistral': 2926, 'n_tokens_neox': 2804, 'n_words': 925}