<commit_before>#!/usr/bin/env python
# -- coding: utf-8 --
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import logging
import os
import subprocess
import uuid

from tempfile import NamedTemporaryFile

from django.utils.translation import ugettext as _

from desktop.lib.exceptions_renderable import PopupException
from desktop.lib import export_csvxls
from desktop.lib.rest.http_client import HttpClient, RestException
from desktop.lib.rest import resource

from metadata.conf import OPTIMIZER, get_optimizer_url
from subprocess import CalledProcessError


LOG = logging.getLogger(__name__)


_JSON_CONTENT_TYPE = 'application/json'


def is_optimizer_enabled():
  return get_optimizer_url() and OPTIMIZER.PRODUCT_NAME.get()


class OptimizerApiException(PopupException):
  pass


class OptimizerApi(object):

  def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
    self._api_url = (api_url or get_optimizer_url()).strip('/')
    self._email = OPTIMIZER.EMAIL.get()
    self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
    self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
    self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get()
    self._product_name = product_name if product_name else (OPTIMIZER.PRODUCT_NAME.get() or self.get_tenant()['tenant']) # Aka "workload"

    self._client = HttpClient(self._api_url, logger=LOG)
    self._client.set_verify(ssl_cert_ca_verify)

    self._root = resource.Resource(self._client)
    self._token = None


  def _authenticate(self, force=False):
    if self._token is None or force:
      self._token = self.authenticate()['token']

    return self._token


  def _exec(self, command, args):
    data = None
    response = {'status': 'error'}

    try:
      cmd_args = [
          'ccs',
          'navopt',
          '--endpoint-url=%s' % self._api_url,
          command
      ]
      if self._product_secret:
        cmd_args += ['--auth-config', self._product_secret]

      LOG.info(' '.join(cmd_args + args))
      data = subprocess.check_output(cmd_args + args)
    except CalledProcessError, e:
      if command == 'upload' and e.returncode == 1:
        LOG.info('Upload command is successful despite return code of 1: %s' % e.output)
        data = '\n'.join(e.output.split('\n')[3:]) # Beware removing of {"url":...}
      else:
        raise OptimizerApiException(e, title=_('Error while accessing Optimizer'))
    except RestException, e:
      raise OptimizerApiException(e, title=_('Error while accessing Optimizer'))

    if data:
      response = json.loads(data)
      if 'status' not in response:
        response['status'] = 'success'
    return response


  def get_tenant(self, email=None):
    return self._exec('get-tenant', ['--email', email or self._email])


  def create_tenant(self, group):
    return self._exec('create-tenant', ['--user-group', group])


  def authenticate(self):
    try:
      data = {
          'productName': self._product_name,
          'productSecret': self._product_secret,
      }
      return self._root.post('/api/authenticate', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
    except RestException, e:
      raise PopupException(e, title=_('Error while accessing Optimizer'))


  def delete_workload(self, token, email=None):
    try:
      data = {
          'email': email if email is not None else self._email,
          'token': token,
      }
      return self._root.post('/api/deleteWorkload', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
    except RestException, e:
      raise PopupException(e, title=_('Error while accessing Optimizer'))


  def get_status(self, token, email=None):
    try:
      data = {
          'email': email if email is not None else self._email,
          'token': token,
      }
      return self._root.post('/api/getStatus', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
    except RestException, e:
      raise PopupException(e, title=_('Error while accessing Optimizer'))


  def upload(self, queries, source_platform='generic', workload_id=None):
    with NamedTemporaryFile(suffix='.csv') as f_queries_path, NamedTemporaryFile(suffix='.json') as f_format_path:
      pass

    try:
      with open(f_queries_path.name, 'w+') as f_queries, open(f_format_path.name, 'w+') as f_format:
        content_generator = OptimizerDataAdapter(queries)
        queries_csv = export_csvxls.create_generator(content_generator, 'csv')

        for row in queries_csv:
          f_queries.write(row)

        f_format.write("""{
    "fileLocation": "%(query_file)s",
    "tenant": "%(tenant)s",
    "fileName": "%(query_file_name)s",
    "sourcePlatform": "hive",
    "colDelim": ",",
    "rowDelim": "\\n",
    "headerFields": [
        {
            "count": 0,
            "coltype": "SQL_ID",
            "use": true,
            "tag": "",
            "name": "SQL_ID"
        },
        {
            "count": 0,
            "coltype": "NONE",
            "use": true,
            "tag": "",
            "name": "ELAPSED_TIME"
        },
        {
            "count": 0,
            "coltype": "SQL_QUERY",
            "use": true,
            "tag": "",
            "name": "SQL_FULLTEXT"
        }
    ]
}""" % {'tenant': self._product_name, 'query_file': f_queries.name, 'query_file_name': os.path.basename(f_queries.name)})

      args = [
          '--cli-input-json', 'file://%s' % f_format.name
      ]
      if workload_id:
        args += ['--workload-id', workload_id]

      return self._exec('upload', args)

    except RestException, e:<commit_msg>HUE-5443 [metadata] Remove with instruction which is not Python 2.4 compatible<commit_after>#!/usr/bin/env python
# -- coding: utf-8 --
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import logging
import os
import subprocess
import uuid

from tempfile import NamedTemporaryFile

from django.utils.translation import ugettext as _

from desktop.lib.exceptions_renderable import PopupException
from desktop.lib import export_csvxls
from desktop.lib.rest.http_client import HttpClient, RestException
from desktop.lib.rest import resource

from metadata.conf import OPTIMIZER, get_optimizer_url
from subprocess import CalledProcessError


LOG = logging.getLogger(__name__)


_JSON_CONTENT_TYPE = 'application/json'


def is_optimizer_enabled():
  return get_optimizer_url() and OPTIMIZER.PRODUCT_NAME.get()


class OptimizerApiException(PopupException):
  pass


class OptimizerApi(object):

  def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
    self._api_url = (api_url or get_optimizer_url()).strip('/')
    self._email = OPTIMIZER.EMAIL.get()
    self._email_password = <PASSWORD>()
    self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
    self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get()
    self._product_name = product_name if product_name else (OPTIMIZER.PRODUCT_NAME.get() or self.get_tenant()['tenant']) # Aka "workload"

    self._client = HttpClient(self._api_url, logger=LOG)
    self._client.set_verify(ssl_cert_ca_verify)

    self._root = resource.Resource(self._client)
    self._token = None


  def _authenticate(self, force=False):
    if self._token is None or force:
      self._token = self.authenticate()['token']

    return self._token


  def _exec(self, command, args):
    data = None
    response = {'status': 'error'}

    try:
      cmd_args = [
          'ccs',
          'navopt',
          '--endpoint-url=%s' % self._api_url,
          command
      ]
      if self._product_secret:
        cmd_args += ['--auth-config', self._product_secret]

      LOG.info(' '.join(cmd_args + args))
      data = subprocess.check_output(cmd_args + args)
    except CalledProcessError, e:
      if command == 'upload' and e.returncode == 1:
        LOG.info('Upload command is successful despite return code of 1: %s' % e.output)
        data = '\n'.join(e.output.split('\n')[3:]) # Beware removing of {"url":...}
      else:
        raise OptimizerApiException(e, title=_('Error while accessing Optimizer'))
    except RestException, e:
      raise OptimizerApiException(e, title=_('Error while accessing Optimizer'))

    if data:
      response = json.loads(data)
      if 'status' not in response:
        response['status'] = 'success'
    return response


  def get_tenant(self, email=None):
    return self._exec('get-tenant', ['--email', email or self._email])


  def create_tenant(self, group):
    return self._exec('create-tenant', ['--user-group', group])


  def authenticate(self):
    try:
      data = {
          'productName': self._product_name,
          'productSecret': self._product_secret,
      }
      return self._root.post('/api/authenticate', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
    except RestException, e:
      raise PopupException(e, title=_('Error while accessing Optimizer'))


  def delete_workload(self, token, email=None):
    try:
      data = {
          'email': email if email is not None else self._email,
          'token': token,
      }
      return self._root.post('/api/deleteWorkload', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
    except RestException, e:
      raise PopupException(e, title=_('Error while accessing Optimizer'))


  def get_status(self, token, email=None):
    try:
      data = {
          'email': email if email is not None else self._email,
          'token': token,
      }
      return self._root.post('/api/getStatus', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
    except RestException, e:
      raise PopupException(e, title=_('Error while accessing Optimizer'))


  def upload(self, queries, source_platform='generic', workload_id=None):
    f_queries_path = NamedTemporaryFile(suffix='.csv')
    f_format_path = NamedTemporaryFile(suffix='.json')
    f_queries_path.close()
    f_format_path.close() # Reopened as real file below to work well with the command

    try:
      f_queries = open(f_queries_path.name, 'w+')
      f_format = open(f_format_path.name, 'w+')

      try:
        content_generator = OptimizerDataAdapter(queries)
        queries_csv = export_csvxls.create_generator(content_generator, 'csv')

        for row in queries_csv:
          f_queries.write(row)

        f_format.write("""{
    "fileLocation": "%(query_file)s",
    "tenant": "%(tenant)s",
    "fileName": "%(query_file_name)s",
    "sourcePlatform": "hive",
    "colDelim": ",",
    "rowDelim": "\\n",
    "headerFields": [
        {
            "count": 0,
            "coltype": "SQL_ID",
            "use": true,
            "tag": "",
            "name": "SQL_ID"
        },
        {
            "count": 0,
            "coltype": "NONE",
            "use": true,
            "tag": "",
            "name": "ELAPSED_TIME"
        },
        {
            "count": 0,
            "coltype": "SQL_QUERY",
            "use": true,
            "tag": "",
            "name": "SQL_FULLTEXT"
        }
    ]
}""" % {'tenant': self._product_name, 'query_file': f_queries.name, 'query_file_name': os.path.basename(f_queries.name)})

      finally:
        f_queries.close()
        f_format.close()

      args = [
          '--cli-input-json', 'file://%s' % f_format.name
      ]
      if workload_id:
        args += ['--workload-id', workload_id]

      return self._exec('upload', args)

    except RestException, e:
{'fraction_non_alphanumeric': 0.11869684082624544, 'fraction_numerical': 0.00212636695018226, 'mean_word_length': 3.1282131661442008, 'pattern_counts': {'":': 46, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6716009', 'n_tokens_mistral': 4107, 'n_tokens_neox': 4016, 'n_words': 1027}