<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import json
import logging
import re

from django.utils.translation import ugettext as _

from desktop.lib.django_util import JsonResponse
from desktop.lib.exceptions_renderable import PopupException
from search.models import Collection

from indexer.controller import CollectionManagerController
from indexer.utils import fields_from_log, field_values_from_separated_file, get_type_from_morphline_type, \
  get_field_types


LOG = logging.getLogger(__name__)


def parse_fields(request):
  if request.method != 'POST':
    raise PopupException(_('POST request required.'))

  result = {'status': -1}

  source_type = request.POST.get('source')

  if source_type == 'file':
    content_type = request.POST.get('type')
    try:
      if content_type == 'separated':
        delimiter = request.POST.get('separator', ',')
        quote = request.POST.get('quote', '"')
        file_obj = request.fs.open(request.POST.get('path'))
        field_list = field_values_from_separated_file(file_obj, delimiter, quote)
        row = next(field_list)
        field_names = row.keys()
        field_types = get_field_types((row.values() for row in itertools.chain([row], field_list)), iterations=51)
        file_obj.close()

        result['data'] = zip(field_names, field_types)
        result['status'] = 0
      elif content_type == 'morphlines':
        morphlines = json.loads(request.POST.get('morphlines'))
        # Look for entries that take on the form %{SYSLOGTIMESTAMP:timestamp}
        field_results = re.findall(r'\%\{(?P<type>\w+)\:(?P<name>\w+)\}', morphlines['expression'])
        if field_results:
          result['data'] = []

          for field_result in field_results:
            result['data'].append( (field_result[1], get_type_from_morphline_type(field_result[0])) )

          result['status'] = 0
        else:
          result['status'] = 1
          result['message'] = _('Could not detect any fields.')
      elif content_type == 'log':
        file_obj = request.fs.open(request.POST.get('path'))
        result['data'] = fields_from_log(file_obj)
        file_obj.close()

        result['status'] = 0
      else:
        result['status'] = 1
        result['message'] = _('Type %s not supported.') % content_type
    except Exception, e:
      LOG.exception(e.message)
      result['message'] = e.message
  else:
    result['message'] = _('Source type %s not supported.') % source_type

  return JsonResponse(result)

def autocomplete(request):
  searcher = CollectionManagerController(request.user)
  autocomplete = searcher.get_autocomplete()

  massaged_collections = []

  for collection in autocomplete['collections']:
    massaged_collections.append({
      'name': collection,
      'isCollection': True,
      'isConfig': False,
    })

  for config in autocomplete['configs']:
    massaged_collections.append({
      'name': config,
      'isCollection': False,
      'isConfig': True,
    })

  response = {
    'status': 0,
    'collections': massaged_collections
  }

  return JsonResponse(response)


def collections(request):
  searcher = CollectionManagerController(request.user)
  solr_collections = searcher.get_collections()
  massaged_collections = []

  for collection in solr_collections:
    massaged_collections.append({
      'name': collection,
      'isCoreOnly': solr_collections[collection]['isCoreOnly'],
      'isAlias': solr_collections[collection].get('isAlias', False),
      'collections': solr_collections[collection].get('collections', []),
    })

  response = {
    'status': 0,
    'collections': massaged_collections
  }

  return JsonResponse(response)


def collections_create(request):
  if request.method != 'POST':
    raise PopupException(_('POST request required.'))

  response = {'status': -1}

  collection = json.loads(request.POST.get('collection', '{}'))

  if collection:
    searcher = CollectionManagerController(request.user)

    # Create instance directory, collection, and add fields
    searcher.create_collection(collection.get('name'), collection.get('fields', []), collection.get('uniqueKeyField'), collection.get('df'))

    try:
      if request.POST.get('source') == 'file':
        # Index data
        searcher.update_data_from_hdfs(request.fs,<commit_msg>HUE-5304 [indexer] Use new API to list indexes<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import json
import logging
import re

from django.utils.translation import ugettext as _

from desktop.lib.django_util import JsonResponse
from desktop.lib.exceptions_renderable import PopupException
from search.models import Collection

from indexer.controller import CollectionManagerController
from indexer.solr_client import SolrClient
from indexer.utils import fields_from_log, field_values_from_separated_file, get_type_from_morphline_type, get_field_types


LOG = logging.getLogger(__name__)


def parse_fields(request):
  if request.method != 'POST':
    raise PopupException(_('POST request required.'))

  result = {'status': -1}

  source_type = request.POST.get('source')

  if source_type == 'file':
    content_type = request.POST.get('type')
    try:
      if content_type == 'separated':
        delimiter = request.POST.get('separator', ',')
        quote = request.POST.get('quote', '"')
        file_obj = request.fs.open(request.POST.get('path'))
        field_list = field_values_from_separated_file(file_obj, delimiter, quote)
        row = next(field_list)
        field_names = row.keys()
        field_types = get_field_types((row.values() for row in itertools.chain([row], field_list)), iterations=51)
        file_obj.close()

        result['data'] = zip(field_names, field_types)
        result['status'] = 0
      elif content_type == 'morphlines':
        morphlines = json.loads(request.POST.get('morphlines'))
        # Look for entries that take on the form %{SYSLOGTIMESTAMP:timestamp}
        field_results = re.findall(r'\%\{(?P<type>\w+)\:(?P<name>\w+)\}', morphlines['expression'])
        if field_results:
          result['data'] = []

          for field_result in field_results:
            result['data'].append( (field_result[1], get_type_from_morphline_type(field_result[0])) )

          result['status'] = 0
        else:
          result['status'] = 1
          result['message'] = _('Could not detect any fields.')
      elif content_type == 'log':
        file_obj = request.fs.open(request.POST.get('path'))
        result['data'] = fields_from_log(file_obj)
        file_obj.close()

        result['status'] = 0
      else:
        result['status'] = 1
        result['message'] = _('Type %s not supported.') % content_type
    except Exception, e:
      LOG.exception(e.message)
      result['message'] = e.message
  else:
    result['message'] = _('Source type %s not supported.') % source_type

  return JsonResponse(result)

def autocomplete(request):
  searcher = CollectionManagerController(request.user)
  autocomplete = searcher.get_autocomplete()

  massaged_collections = []

  for collection in autocomplete['collections']:
    massaged_collections.append({
      'name': collection,
      'isCollection': True,
      'isConfig': False,
    })

  for config in autocomplete['configs']:
    massaged_collections.append({
      'name': config,
      'isCollection': False,
      'isConfig': True,
    })

  response = {
    'status': 0,
    'collections': massaged_collections
  }

  return JsonResponse(response)


def collections(request):
  client = SolrClient(user=request.user)

  response = {
    'status': 0,
    'collections': client.get_indexes()
  }

  return JsonResponse(response)


def collections_create(request):
  if request.method != 'POST':
    raise PopupException(_('POST request required.'))

  response = {'status': -1}

  collection = json.loads(request.POST.get('collection', '{}'))

  if collection:
    searcher = CollectionManagerController(request.user)

    # Create instance directory, collection, and add fields
    searcher.create_collection(collection.get('name'), collection.get('fields', []), collection.get('uniqueKeyField'), collection.get('df'))

    try:
      if request.POST.get('source') == 'file':
        # Index data
        searcher.update_data_from_hdfs(request.fs,
{'fraction_non_alphanumeric': 0.11831789023521026, 'fraction_numerical': 0.0038692597495163425, 'mean_word_length': 3.704022988505747, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5511884', 'n_tokens_mistral': 2881, 'n_tokens_neox': 2812, 'n_words': 739}