<commit_before>#!/usr/bin/env python
# -- coding: utf-8 --
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import csv
import logging
import os
import pytz
import re
import shutil
import StringIO
import tempfile
import uuid

from dateutil.parser import parse

from django.conf import settings
from django.utils.translation import ugettext as _

from desktop.lib.i18n import force_unicode, smart_str

from indexer import conf
from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES,\
                           DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES

LOG = logging.getLogger(__name__)
TIMESTAMP_PATTERN = '\[([\w\d\s\-\/\:\+]*?)\]'
FIELD_XML_TEMPLATE = '<field name="%(name)s" type="%(type)s" indexed="%(indexed)s" stored="%(stored)s" required="%(required)s" />'
DEFAULT_FIELD = {
  'name': None,
  'type': 'text',
  'indexed': 'true',
  'stored': 'true',
  'required': 'true'
}

def get_config_template_path(solr_cloud_mode):
  if solr_cloud_mode:
    return os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'solrcloud')
  else:
    return os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'nonsolrcloud')


class SchemaXml(object):
  def __init__(self, xml):
    self.xml = xml

  def uniqueKeyField(self, unique_key_field):
    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE UNIQUE KEY -->', force_unicode(unique_key_field)))

  def fields(self, fields):
    fields_xml = ''
    for field in fields:
      field_dict = DEFAULT_FIELD.copy()
      field_dict.update(field)
      fields_xml += FIELD_XML_TEMPLATE % field_dict + '\n'
    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE FIELDS -->', force_unicode(fields_xml)))


class SolrConfigXml(object):
  def __init__(self, xml):
    self.xml = xml

  def defaultField(self, df):
    self.xml = force_unicode(force_unicode(self.xml).replace(u'<str name="df">text</str>', u'<str name="df">%s</str>' % force_unicode(df)))


def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
  # Create temporary copy of solr configs
  tmp_path = tempfile.mkdtemp()

  config_template_path = get_config_template_path(solr_cloud_mode)

  solr_config_path = os.path.join(tmp_path, 'solr_configs')
  shutil.copytree(config_template_path, solr_config_path)

  if fields or unique_key_field:
    # Get complete schema.xml
    with open(os.path.join(config_template_path, 'conf/schema.xml')) as f:
      schemaxml = SchemaXml(f.read())
      schemaxml.uniqueKeyField(unique_key_field)
      schemaxml.fields(fields)

    # Write complete schema.xml to copy
    with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
      f.write(smart_str(schemaxml.xml))

  if df:
    # Get complete solrconfig.xml
    with open(os.path.join(config_template_path, 'conf/solrconfig.xml')) as f:
      solrconfigxml = SolrConfigXml(f.read())
      solrconfigxml.defaultField(df)

    # Write complete solrconfig.xml to copy
    with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
      f.write(smart_str(solrconfigxml.xml))

  return tmp_path, solr_config_path


def get_field_types(field_list, iterations=3):
  assert iterations > 0, "iterations should be a positive integer (not a negative integer or 0)"

  def test_boolean(value):
    if value.lower() not in ('false', 'true'):
      raise ValueError(_("%s is not a boolean value") % value)

  def test_timestamp(value):
    if not value:
      raise ValueError()

    if len(value) > 50 or len(value) < 3:
      raise ValueError()

    if value.startswith('[') and value.endswith(']'):
      value = value[1:-1]

    try:
      parse(value)
    except:
      LOG.exception('failed to parse value %s' % value)
      raise ValueError()

  def test_int(value):
    if len(bin(int(value))) - 2 > 32:
      raise ValueError()

  def test_string(value):
    if len(smart_str(value).split(' ')) > 4:
      raise ValueError()

  test_fns = [('boolean', test_boolean),
              ('tint', test_int),<commit_msg>[indexer] dateutil.parse only throws ValueError or OverflowError<commit_after>#!/usr/bin/env python
# -- coding: utf-8 --
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import csv
import logging
import os
import pytz
import re
import shutil
import StringIO
import tempfile
import uuid

from dateutil.parser import parse

from django.conf import settings
from django.utils.translation import ugettext as _

from desktop.lib.i18n import force_unicode, smart_str

from indexer import conf
from indexer.models import DATE_FIELD_TYPES, TEXT_FIELD_TYPES, INTEGER_FIELD_TYPES,\
                           DECIMAL_FIELD_TYPES, BOOLEAN_FIELD_TYPES

LOG = logging.getLogger(__name__)
TIMESTAMP_PATTERN = '\[([\w\d\s\-\/\:\+]*?)\]'
FIELD_XML_TEMPLATE = '<field name="%(name)s" type="%(type)s" indexed="%(indexed)s" stored="%(stored)s" required="%(required)s" />'
DEFAULT_FIELD = {
  'name': None,
  'type': 'text',
  'indexed': 'true',
  'stored': 'true',
  'required': 'true'
}

def get_config_template_path(solr_cloud_mode):
  if solr_cloud_mode:
    return os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'solrcloud')
  else:
    return os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'nonsolrcloud')


class SchemaXml(object):
  def __init__(self, xml):
    self.xml = xml

  def uniqueKeyField(self, unique_key_field):
    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE UNIQUE KEY -->', force_unicode(unique_key_field)))

  def fields(self, fields):
    fields_xml = ''
    for field in fields:
      field_dict = DEFAULT_FIELD.copy()
      field_dict.update(field)
      fields_xml += FIELD_XML_TEMPLATE % field_dict + '\n'
    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE FIELDS -->', force_unicode(fields_xml)))


class SolrConfigXml(object):
  def __init__(self, xml):
    self.xml = xml

  def defaultField(self, df):
    self.xml = force_unicode(force_unicode(self.xml).replace(u'<str name="df">text</str>', u'<str name="df">%s</str>' % force_unicode(df)))


def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
  # Create temporary copy of solr configs
  tmp_path = tempfile.mkdtemp()

  config_template_path = get_config_template_path(solr_cloud_mode)

  solr_config_path = os.path.join(tmp_path, 'solr_configs')
  shutil.copytree(config_template_path, solr_config_path)

  if fields or unique_key_field:
    # Get complete schema.xml
    with open(os.path.join(config_template_path, 'conf/schema.xml')) as f:
      schemaxml = SchemaXml(f.read())
      schemaxml.uniqueKeyField(unique_key_field)
      schemaxml.fields(fields)

    # Write complete schema.xml to copy
    with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
      f.write(smart_str(schemaxml.xml))

  if df:
    # Get complete solrconfig.xml
    with open(os.path.join(config_template_path, 'conf/solrconfig.xml')) as f:
      solrconfigxml = SolrConfigXml(f.read())
      solrconfigxml.defaultField(df)

    # Write complete solrconfig.xml to copy
    with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
      f.write(smart_str(solrconfigxml.xml))

  return tmp_path, solr_config_path


def get_field_types(field_list, iterations=3):
  assert iterations > 0, "iterations should be a positive integer (not a negative integer or 0)"

  def test_boolean(value):
    if value.lower() not in ('false', 'true'):
      raise ValueError(_("%s is not a boolean value") % value)

  def test_timestamp(value):
    if not value:
      raise ValueError()

    if len(value) > 50 or len(value) < 3:
      raise ValueError()

    if value.startswith('[') and value.endswith(']'):
      value = value[1:-1]

    try:
      parse(value)
    except OverflowError:
      raise ValueError()

  def test_int(value):
    if len(bin(int(value))) - 2 > 32:
      raise ValueError()

  def test_string(value):
    if len(smart_str(value).split(' ')) > 4:
      raise ValueError()

  test_fns = [('boolean', test_boolean),
              ('tint', test_int),
{'fraction_non_alphanumeric': 0.11993654151242729, 'fraction_numerical': 0.004019037546271814, 'mean_word_length': 4.086605701990317, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 64}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7236910', 'n_tokens_mistral': 3174, 'n_tokens_neox': 3083, 'n_words': 816}