<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from builtins import str
from builtins import object
import logging
import re
import threading
import time
import json

from django.core.cache import caches
from django.urls import reverse
from django.utils.encoding import force_unicode
from django.utils.translation import ugettext as _
from kazoo.client import KazooClient

from desktop.conf import CLUSTER_ID
from desktop.lib.django_util import format_preserving_redirect
from desktop.lib.exceptions_renderable import PopupException
from desktop.lib.parameterization import substitute_variables
from desktop.lib.view_util import location_to_url
from desktop.models import Cluster
from desktop.settings import CACHES_HIVE_DISCOVERY_KEY
from indexer.file_format import HiveFormat
from libzookeeper import conf as libzookeeper_conf

from beeswax import hive_site
from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_METASTORE_HOST, HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
  AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
  HIVE_DISCOVERY_HS2, HIVE_DISCOVERY_LLAP, HIVE_DISCOVERY_LLAP_HA, HIVE_DISCOVERY_LLAP_ZNODE, CACHE_TIMEOUT, \
  LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT
from beeswax.common import apply_natural_sort
from beeswax.design import hql_query
from beeswax.hive_site import hiveserver2_use_ssl
from beeswax.models import QueryHistory, QUERY_TYPES


LOG = logging.getLogger(__name__)


DBMS_CACHE = {}
DBMS_CACHE_LOCK = threading.Lock()
cache = caches[CACHES_HIVE_DISCOVERY_KEY]
# Using file cache to make sure eventlet threads are uniform, this cache is persistent on startup
# So we clear it to make sure the server resets hiveserver2 host.
cache.clear()

def get(user, query_server=None, cluster=None):
  global DBMS_CACHE
  global DBMS_CACHE_LOCK

  if query_server is None:
    query_server = get_query_server_config(connector=cluster)

  DBMS_CACHE_LOCK.acquire()
  try:
    DBMS_CACHE.setdefault(user.id, {})

    if query_server['server_name'] not in DBMS_CACHE[user.id]:
      # Avoid circular dependency
      from beeswax.server.hive_server2_lib import HiveServerClientCompatible

      if query_server['server_name'].startswith('impala'):
        from impala.dbms import ImpalaDbms
        from impala.server import ImpalaServerClient
        DBMS_CACHE[user.id][query_server['server_name']] = ImpalaDbms(HiveServerClientCompatible(ImpalaServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
      elif query_server['server_name'] == 'hms':
        from beeswax.server.hive_metastore_server import HiveMetastoreClient
        DBMS_CACHE[user.id][query_server['server_name']] = HiveServer2Dbms(HiveMetastoreClient(query_server, user), QueryHistory.SERVER_TYPE[1][0])
      else:
        from beeswax.server.hive_server2_lib import HiveServerClient
        DBMS_CACHE[user.id][query_server['server_name']] = HiveServer2Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])

    return DBMS_CACHE[user.id][query_server['server_name']]
  finally:
    DBMS_CACHE_LOCK.release()


def get_query_server_config(name='beeswax', connector=None):
  if connector:
    connector_name = full_connector_name = connector['type']
    compute_name = None
    if connector.get('compute'):
      compute_name = connector['compute']['name']
      full_connector_name = '%s-%s' % (connector_name, compute_name)
    LOG.debug("Query cluster connector %s compute %s" % (connector_name, compute_name))

    query_server = {
        'server_name': full_connector_name,
        'server_host': (connector['compute']['options'] if 'compute' in connector else connector['options'])['server_host'],
        'server_port': int((connector['compute']['options'] if 'compute' in connector else connector['options'])['server_port']),
        'principal': 'TODO',
        'auth_username': AUTH_USERNAME.get(),
        'auth_password': <PASSWORD>(),<commit_msg>HUE-8758 [connectors] Use connector server_config when has_connectors<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from builtins import str
from builtins import object
import logging
import re
import threading
import time
import json

from django.core.cache import caches
from django.urls import reverse
from django.utils.encoding import force_unicode
from django.utils.translation import ugettext as _
from kazoo.client import KazooClient

from desktop.conf import CLUSTER_ID, has_connectors
from desktop.lib.django_util import format_preserving_redirect
from desktop.lib.exceptions_renderable import PopupException
from desktop.lib.parameterization import substitute_variables
from desktop.lib.view_util import location_to_url
from desktop.models import Cluster
from desktop.settings import CACHES_HIVE_DISCOVERY_KEY
from indexer.file_format import HiveFormat
from libzookeeper import conf as libzookeeper_conf

from beeswax import hive_site
from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, HIVE_METASTORE_HOST, HIVE_METASTORE_PORT, LIST_PARTITIONS_LIMIT, SERVER_CONN_TIMEOUT, \
  AUTH_USERNAME, AUTH_PASSWORD, APPLY_NATURAL_SORT_MAX, QUERY_PARTITIONS_LIMIT, HIVE_DISCOVERY_HIVESERVER2_ZNODE, \
  HIVE_DISCOVERY_HS2, HIVE_DISCOVERY_LLAP, HIVE_DISCOVERY_LLAP_HA, HIVE_DISCOVERY_LLAP_ZNODE, CACHE_TIMEOUT, \
  LLAP_SERVER_HOST, LLAP_SERVER_PORT, LLAP_SERVER_THRIFT_PORT
from beeswax.common import apply_natural_sort
from beeswax.design import hql_query
from beeswax.hive_site import hiveserver2_use_ssl
from beeswax.models import QueryHistory, QUERY_TYPES


LOG = logging.getLogger(__name__)


DBMS_CACHE = {}
DBMS_CACHE_LOCK = threading.Lock()
cache = caches[CACHES_HIVE_DISCOVERY_KEY]
# Using file cache to make sure eventlet threads are uniform, this cache is persistent on startup
# So we clear it to make sure the server resets hiveserver2 host.
cache.clear()

def get(user, query_server=None, cluster=None):
  global DBMS_CACHE
  global DBMS_CACHE_LOCK

  if query_server is None:
    query_server = get_query_server_config(connector=cluster)

  DBMS_CACHE_LOCK.acquire()
  try:
    DBMS_CACHE.setdefault(user.id, {})

    if query_server['server_name'] not in DBMS_CACHE[user.id]:
      # Avoid circular dependency
      from beeswax.server.hive_server2_lib import HiveServerClientCompatible

      if query_server['server_name'].startswith('impala'):
        from impala.dbms import ImpalaDbms
        from impala.server import ImpalaServerClient
        DBMS_CACHE[user.id][query_server['server_name']] = ImpalaDbms(HiveServerClientCompatible(ImpalaServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
      elif query_server['server_name'] == 'hms':
        from beeswax.server.hive_metastore_server import HiveMetastoreClient
        DBMS_CACHE[user.id][query_server['server_name']] = HiveServer2Dbms(HiveMetastoreClient(query_server, user), QueryHistory.SERVER_TYPE[1][0])
      else:
        from beeswax.server.hive_server2_lib import HiveServerClient
        DBMS_CACHE[user.id][query_server['server_name']] = HiveServer2Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])

    return DBMS_CACHE[user.id][query_server['server_name']]
  finally:
    DBMS_CACHE_LOCK.release()


def get_query_server_config(name='beeswax', connector=None):
  if connector and has_connectors(): # FIXME: Currently this doesn't work when has_connectors is False because options is empty
    connector_name = full_connector_name = connector['type']
    compute_name = None
    if connector.get('compute'):
      compute_name = connector['compute']['name']
      full_connector_name = '%s-%s' % (connector_name, compute_name)
    LOG.debug("Query cluster connector %s compute %s" % (connector_name, compute_name))

    query_server = {
        'server_name': full_connector_name,
        'server_host': (connector['compute']['options'] if 'compute' in connector else connector['options'])['server_host'],
        'server_port': int((connector['compute']['options'] if 'compute' in connector else connector['options'])['server_port']),
        'principal': 'TODO',
        'auth_username': AUTH_USERNAME.get(),
        'auth_password': AUTH_PASSWORD.get(),
{'fraction_non_alphanumeric': 0.08059732024138283, 'fraction_numerical': 0.004091234530019434, 'mean_word_length': 5.562416107382551, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7135812', 'n_tokens_mistral': 3022, 'n_tokens_neox': 2966, 'n_words': 791}