<commit_before># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: <EMAIL>
# Maintained By: <EMAIL>

from dateutil.parser import parse
from flask import current_app
from sqlalchemy import and_
from sqlalchemy import or_
import re
import traceback

from ggrc import db
from ggrc.automapper import AutomapperGenerator
from ggrc.converters import get_importables
from ggrc.converters import errors
from ggrc.login import get_current_user
from ggrc.models import Audit
from ggrc.models import AuditObject
from ggrc.models import CategoryBase
from ggrc.models import CustomAttributeDefinition
from ggrc.models import CustomAttributeValue
from ggrc.models import ObjectPerson
from ggrc.models import Option
from ggrc.models import Person
from ggrc.models import Policy
from ggrc.models import Program
from ggrc.models import Regulation
from ggrc.models import Relationship
from ggrc.models import Request
from ggrc.models import Response
from ggrc.models import Standard
from ggrc.models import all_models
from ggrc.models.relationship_helper import RelationshipHelper
from ggrc.rbac import permissions
from ggrc.models.reflection import AttributeInfo


MAPPING_PREFIX = "__mapping__:"
CUSTOM_ATTR_PREFIX = "__custom__:"


class ColumnHandler(object):

  def __init__(self, row_converter, key, **options):
    self.row_converter = row_converter
    self.key = key
    self.value = None
    self.raw_value = options.get("raw_value", "").strip()
    self.validator = options.get("validator")
    self.mandatory = options.get("mandatory", False)
    self.default = options.get("default")
    self.description = options.get("description", "")
    self.display_name = options.get("display_name", "")
    self.dry_run = row_converter.block_converter.converter.dry_run
    self.new_objects = self.row_converter.block_converter.converter.new_objects
    self.set_value()

  def set_value(self):
    self.value = self.parse_item()

  def get_value(self):
    return getattr(self.row_converter.obj, self.key, self.value)

  def add_error(self, template, **kwargs):
    self.row_converter.add_error(template, **kwargs)

  def add_warning(self, template, **kwargs):
    self.row_converter.add_warning(template, **kwargs)

  def parse_item(self):
    return self.raw_value

  def set_obj_attr(self):
    if not self.value:
      return
    try:
      setattr(self.row_converter.obj, self.key, self.value)
    except:
      self.row_converter.add_error(errors.UNKNOWN_ERROR)
      trace = traceback.format_exc()
      error = "Import failed with:\nsetattr({}, {}, {})\n{}".format(
          self.row_converter.obj, self.key, self.value, trace)
      current_app.logger.error(error)

  def get_default(self):
    if callable(self.default):
      return self.default()
    return self.default

  def insert_object(self):
    """ For inserting fields such as custom attributes and mappings """
    pass


class DeleteColumnHandler(ColumnHandler):

  # this is a white list of objects that can be deleted in a cascade
  # e.g. deleting a Market can delete the associated ObjectOwner objectect too
  delete_whitelist = {"Relationship", "ObjectOwner", "ObjectPerson"}

  def get_value(self):
    return ""

  def parse_item(self):
    is_delete = self.raw_value.lower() in ["true", "yes"]
    self.row_converter.is_delete = is_delete
    return is_delete

  def set_obj_attr(self):
    if not self.value:
      return
    obj = self.row_converter.obj
    if self.row_converter.is_new:
      self.add_error(errors.DELETE_NEW_OBJECT_ERROR,
                     object_type=obj.type,
                     slug=obj.slug)
      return
    if self.row_converter.ignore:
      return
    tr = db.session.begin_nested()
    try:
      tr.session.delete(obj)
      deleted = len([o for o in tr.session.deleted
                    if o.type not in self.delete_whitelist])
      if deleted != 1:
        self.add_error(errors.DELETE_CASCADE_ERROR,
                       object_type=obj.type, slug=obj.slug)
    finally:
      if self.dry_run or self.row_converter.ignore:
        tr.rollback()
      else:
        indexer = self.row_converter.block_converter.converter.indexer
        if indexer is not None:
          for o in tr.session.deleted:
            indexer.delete_record(o.id, o.__class__.__name__, commit=False)
        tr.commit()


class StatusColumnHandler(ColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.key = key
    valid_states = row_converter.object_class.VALID_STATES
    self.state_mappings = {str(s).lower(): s for s in valid_states}
    super(StatusColumnHandler, self).__init__(row_converter, key, **options)

  def parse_item(self):
    # TODO: check if mandatory and replace with default if it's wrong
    value = self.raw_value.lower()
    status = self.state_mappings.get(value)
    if status is None:
      self.add_warning(errors.WRONG_REQUIRED_VALUE,
                       value=value[:20],
                       column_name=self.display_name)
      status = self.get_default()
    return status


class UserColumnHandler(ColumnHandler):

  """ Handler for primary and secondary contacts """

  def get_users_list(self):
    users = set()<commit_msg>Hide status warnings on import<commit_after># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: <EMAIL>
# Maintained By: <EMAIL>

from dateutil.parser import parse
from flask import current_app
from sqlalchemy import and_
from sqlalchemy import or_
import re
import traceback

from ggrc import db
from ggrc.automapper import AutomapperGenerator
from ggrc.converters import get_importables
from ggrc.converters import errors
from ggrc.login import get_current_user
from ggrc.models import Audit
from ggrc.models import AuditObject
from ggrc.models import CategoryBase
from ggrc.models import CustomAttributeDefinition
from ggrc.models import CustomAttributeValue
from ggrc.models import ObjectPerson
from ggrc.models import Option
from ggrc.models import Person
from ggrc.models import Policy
from ggrc.models import Program
from ggrc.models import Regulation
from ggrc.models import Relationship
from ggrc.models import Request
from ggrc.models import Response
from ggrc.models import Standard
from ggrc.models import all_models
from ggrc.models.relationship_helper import RelationshipHelper
from ggrc.rbac import permissions
from ggrc.models.reflection import AttributeInfo


MAPPING_PREFIX = "__mapping__:"
CUSTOM_ATTR_PREFIX = "__custom__:"


class ColumnHandler(object):

  def __init__(self, row_converter, key, **options):
    self.row_converter = row_converter
    self.key = key
    self.value = None
    self.raw_value = options.get("raw_value", "").strip()
    self.validator = options.get("validator")
    self.mandatory = options.get("mandatory", False)
    self.default = options.get("default")
    self.description = options.get("description", "")
    self.display_name = options.get("display_name", "")
    self.dry_run = row_converter.block_converter.converter.dry_run
    self.new_objects = self.row_converter.block_converter.converter.new_objects
    self.set_value()

  def set_value(self):
    self.value = self.parse_item()

  def get_value(self):
    return getattr(self.row_converter.obj, self.key, self.value)

  def add_error(self, template, **kwargs):
    self.row_converter.add_error(template, **kwargs)

  def add_warning(self, template, **kwargs):
    self.row_converter.add_warning(template, **kwargs)

  def parse_item(self):
    return self.raw_value

  def set_obj_attr(self):
    if not self.value:
      return
    try:
      setattr(self.row_converter.obj, self.key, self.value)
    except:
      self.row_converter.add_error(errors.UNKNOWN_ERROR)
      trace = traceback.format_exc()
      error = "Import failed with:\nsetattr({}, {}, {})\n{}".format(
          self.row_converter.obj, self.key, self.value, trace)
      current_app.logger.error(error)

  def get_default(self):
    if callable(self.default):
      return self.default()
    return self.default

  def insert_object(self):
    """ For inserting fields such as custom attributes and mappings """
    pass


class DeleteColumnHandler(ColumnHandler):

  # this is a white list of objects that can be deleted in a cascade
  # e.g. deleting a Market can delete the associated ObjectOwner objectect too
  delete_whitelist = {"Relationship", "ObjectOwner", "ObjectPerson"}

  def get_value(self):
    return ""

  def parse_item(self):
    is_delete = self.raw_value.lower() in ["true", "yes"]
    self.row_converter.is_delete = is_delete
    return is_delete

  def set_obj_attr(self):
    if not self.value:
      return
    obj = self.row_converter.obj
    if self.row_converter.is_new:
      self.add_error(errors.DELETE_NEW_OBJECT_ERROR,
                     object_type=obj.type,
                     slug=obj.slug)
      return
    if self.row_converter.ignore:
      return
    tr = db.session.begin_nested()
    try:
      tr.session.delete(obj)
      deleted = len([o for o in tr.session.deleted
                     if o.type not in self.delete_whitelist])
      if deleted != 1:
        self.add_error(errors.DELETE_CASCADE_ERROR,
                       object_type=obj.type, slug=obj.slug)
    finally:
      if self.dry_run or self.row_converter.ignore:
        tr.rollback()
      else:
        indexer = self.row_converter.block_converter.converter.indexer
        if indexer is not None:
          for o in tr.session.deleted:
            indexer.delete_record(o.id, o.__class__.__name__, commit=False)
        tr.commit()


class StatusColumnHandler(ColumnHandler):

  def __init__(self, row_converter, key, **options):
    self.key = key
    valid_states = row_converter.object_class.VALID_STATES
    self.state_mappings = {str(s).lower(): s for s in valid_states}
    super(StatusColumnHandler, self).__init__(row_converter, key, **options)

  def parse_item(self):
    # TODO: check if mandatory and replace with default if it's wrong
    value = self.raw_value.lower()
    status = self.state_mappings.get(value)
    if status is None:
      status = self.get_default()
      if value:
        self.add_warning(errors.WRONG_REQUIRED_VALUE,
                         value=value[:20],
                         column_name=self.display_name)
    return status


class UserColumnHandler(ColumnHandler):

  """ Handler for primary and secondary contacts """

  def get_users_list(self):
    users = set()
{'fraction_non_alphanumeric': 0.0920285768001504, 'fraction_numerical': 0.001692047377326565, 'mean_word_length': 3.5233843537414966, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '948812', 'n_tokens_mistral': 3320, 'n_tokens_neox': 3231, 'n_words': 831}