<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import time

from nose.plugins.skip import SkipTest
from nose.tools import assert_equal, assert_true, assert_false

from django.contrib.auth.models import User

from desktop.auth.backend import rewrite_user
from desktop.lib.django_test_util import make_logged_in_client
from desktop.lib.test_utils import add_to_group, grant_access

from metadata.optimizer_client import OptimizerApi
from metadata.conf import OPTIMIZER, has_optimizer
from desktop.models import uuid_default


LOG = logging.getLogger(__name__)


class BaseTestOptimizerApi(object):

  @classmethod
  def setup_class(cls):
    if not has_optimizer():
      raise SkipTest

    cls.client = make_logged_in_client(username='test', is_superuser=False)
    cls.user = User.objects.get(username='test')
    cls.user = rewrite_user(cls.user)
    add_to_group('test')
    grant_access("test", "test", "metadata")
    grant_access("test", "test", "optimizer")

    cls.api = OptimizerApi(user=cls.user)
    cls.upload()


  # Should run first
  @classmethod
  def upload(cls):
    queries = [
        (uuid_default(), 0, "select emps.id from emps where emps.name = 'Joe' group by emps.mgr, emps.id;", 'db1'),
        (uuid_default(), 0, "select emps.name from emps where emps.num = 007 group by emps.state, emps.name;", 'db1'),
        (uuid_default(), 0, "select Part.partkey, max(Part.salary), Part.name, Part.type from db1.Part where Part.yyprice > 2095", 'db1'),
        (uuid_default(), 0, "select Part.partkey, Part.name, Part.mfgr FROM Part WHERE Part.name LIKE '%red';", 'db1'),
        (uuid_default(), 0, "select count(*) as loans from account a where a.account_state_id in (5,9);", 'db1'),
        (uuid_default(), 0, "select orders.key, orders.id from orders where orders.price < 9999", 'db1'),

        (uuid_default(), 0, "select x from x join y where x.a = y.a;", 'default'),

        # DDL
        (uuid_default(), 0, '''CREATE TABLE `web_logs`(
  `_version_` bigint,
  `app` string COMMENT 'app',
  `bytes` smallint COMMENT 'http://demo.gethue.com/ is',
  `city` string COMMENT 'city',
  `client_ip` string,
  `code` tinyint,
  `country_code` string,
  `country_code3` string,
  `country_name` string,
  `device_family` string,
  `extension` string,
  `latitude` float,
  `longitude` float,
  `method` string,
  `os_family` string,
  `os_major` string,
  `protocol` string,
  `record` string,
  `referer` string,
  `region_code` bigint,
  `request` string,
  `subapp` string,
  `time` string,
  `url` string,
  `user_agent` string,
  `user_agent_family` string,
  `user_agent_major` string,
  `id` string)
COMMENT 'http://demo.gethue.com/ rocks!'
PARTITIONED BY (
  `date` string)
''', 'edw')
    ]<commit_msg>HUE-6335 [optimizer] Strip newlines in test query to not fail at upload<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import time

from nose.plugins.skip import SkipTest
from nose.tools import assert_equal, assert_true, assert_false

from django.contrib.auth.models import User

from desktop.auth.backend import rewrite_user
from desktop.lib.django_test_util import make_logged_in_client
from desktop.lib.test_utils import add_to_group, grant_access

from metadata.optimizer_client import OptimizerApi
from metadata.conf import OPTIMIZER, has_optimizer
from desktop.models import uuid_default


LOG = logging.getLogger(__name__)


class BaseTestOptimizerApi(object):

  @classmethod
  def setup_class(cls):
    if not has_optimizer():
      raise SkipTest

    cls.client = make_logged_in_client(username='test', is_superuser=False)
    cls.user = User.objects.get(username='test')
    cls.user = rewrite_user(cls.user)
    add_to_group('test')
    grant_access("test", "test", "metadata")
    grant_access("test", "test", "optimizer")

    cls.api = OptimizerApi(user=cls.user)
    cls.upload()


  # Should run first
  @classmethod
  def upload(cls):
    queries = [
        (uuid_default(), 0, "select emps.id from emps where emps.name = 'Joe' group by emps.mgr, emps.id;", 'db1'),
        (uuid_default(), 0, "select emps.name from emps where emps.num = 007 group by emps.state, emps.name;", 'db1'),
        (uuid_default(), 0, "select Part.partkey, max(Part.salary), Part.name, Part.type from db1.Part where Part.yyprice > 2095", 'db1'),
        (uuid_default(), 0, "select Part.partkey, Part.name, Part.mfgr FROM Part WHERE Part.name LIKE '%red';", 'db1'),
        (uuid_default(), 0, "select count(*) as loans from account a where a.account_state_id in (5,9);", 'db1'),
        (uuid_default(), 0, "select orders.key, orders.id from orders where orders.price < 9999", 'db1'),

        (uuid_default(), 0, "select x from x join y where x.a = y.a;", 'default'),

        # DDL
        (uuid_default(), 0, ' '.join('''CREATE TABLE `web_logs`(
  `_version_` bigint,
  `app` string COMMENT 'app',
  `bytes` smallint COMMENT 'http://demo.gethue.com/ is',
  `city` string COMMENT 'city',
  `client_ip` string,
  `code` tinyint,
  `country_code` string,
  `country_code3` string,
  `country_name` string,
  `device_family` string,
  `extension` string,
  `latitude` float,
  `longitude` float,
  `method` string,
  `os_family` string,
  `os_major` string,
  `protocol` string,
  `record` string,
  `referer` string,
  `region_code` bigint,
  `request` string,
  `subapp` string,
  `time` string,
  `url` string,
  `user_agent` string,
  `user_agent_family` string,
  `user_agent_major` string,
  `id` string)
COMMENT 'http://demo.gethue.com/ rocks!'
PARTITIONED BY (
  `date` string)
'''.splitlines()), 'edw')
    ]
{'fraction_non_alphanumeric': 0.11909795630725863, 'fraction_numerical': 0.009866102889358703, 'mean_word_length': 4.149492017416546, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7247852', 'n_tokens_mistral': 2324, 'n_tokens_neox': 2263, 'n_words': 730}