<commit_before># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: <EMAIL>
# Maintained By: <EMAIL>

from collections import defaultdict
from collections import OrderedDict
from collections import Counter

from flask import current_app
from sqlalchemy import or_
from sqlalchemy import and_

from ggrc import db
from ggrc import models
from ggrc.utils import benchmark
from ggrc.converters import errors
from ggrc.converters import get_shared_unique_rules
from ggrc.converters.base_row import RowConverter
from ggrc.converters.import_helper import get_column_order
from ggrc.converters.import_helper import get_object_column_definitions
from ggrc.rbac import permissions
from ggrc.services.common import get_modified_objects
from ggrc.services.common import update_index
from ggrc.services.common import update_memcache_after_commit
from ggrc.services.common import update_memcache_before_commit


CACHE_EXPIRY_IMPORT = 600


class BlockConverter(object):

  """ Main block converter class for dealing with csv files and data

  Attributes:
    attr_index (dict): reverse index for getting attribute name from
      display_name
    block_errors (list of str): list containing fatal import errors
    block_warnings (list of str): list containing blokc level import warnings
    row_errors (list of str): list containing row errors
    row_warnings (list of str): list containing row warnings
    object_ids (list of int): list containing all ids for the converted objects
    rows (list of list of str): 2D array containg csv data
    row_converters (list of RowConverter): list of row convertor objects with
      data from the coresponding row in rows attribute
    object_headers (dict): A dictionary containing object headers
    headers (dict): A dictionary containing csv headers with additional
      information. Keys are object attributes such as "title", "slug"...
      Example:
        {"slug": {
            "display_name": "Code",
            "mandatory": True,
            "default": "value or a callabe function that returns the value",
            "validator": validator_function,
            "handler": HandlerClass,
            "description": "description or function that returns description"
            "valid_values": "list of valid values"

  """

  def get_unique_counts_dict(self, object_class):
    """ get a the varible for storing unique counts

    Make sure to always return the same variable for object with shared tables,
    as defined in sharing rules.
    """
    sharing_rules = get_shared_unique_rules()
    classes = sharing_rules.get(object_class, object_class)
    shared_state = self.converter.shared_state
    if classes not in shared_state:
      shared_state[classes] = defaultdict(lambda: defaultdict(list))
    return shared_state[classes]

  def __init__(self, converter, **options):
    self._mapping_cache = None
    self.converter = converter
    self.offset = options.get("offset", 0)
    self.object_class = options.get("object_class")
    self.rows = options.get("rows", [])
    self.object_ids = options.get("object_ids", [])
    self.block_errors = []
    self.block_warnings = []
    self.row_errors = []
    self.row_warnings = []
    self.row_converters = []
    self.ignore = False
    if not self.object_class:
      class_name = options.get("class_name", "")
      self.add_errors(errors.WRONG_OBJECT_TYPE, line=self.offset + 2,
                      object_name=class_name)
      self.name = ""
      return
    self.object_headers = get_object_column_definitions(self.object_class)
    all_header_names = [unicode(key) for key in self.get_header_names().keys()]
    raw_headers = options.get("raw_headers", all_header_names)
    self.check_for_duplicate_columns(raw_headers)
    self.headers = self.clean_headers(raw_headers)
    self.unique_counts = self.get_unique_counts_dict(self.object_class)
    self.name = self.object_class._inflector.human_singular.title()
    self.organize_fields(options.get("fields", []))
    self.ca_definitions_cache = self._create_ca_definitions_cache()

  def _create_ca_definitions_cache(self):
    cad = models.CustomAttributeDefinition
    definition_type = self.object_class._inflector.table_singular
    defs = cad.eager_query().filter(cad.definition_type == definition_type)
    return {(d.definition_id, d.title): d for d in defs}

  def _create_mapping_cache(self):
    def identifier(obj):
      return getattr(obj, "slug", getattr(obj, "email", None))

    relationship = models.Relationship

    with benchmark("cache for: {}".format(self.object_class.__name__)):
      with benchmark("cache query"):
        relationships = relationship.eager_query().filter(or_(
            and_(
                relationship.source_type == self.object_class.__name__,
                relationship.source_id.in_(self.object_ids),
            ),
            and_(
                relationship.destination_type == self.object_class.__name__,
                relationship.destination_id.in_(self.object_ids),
            )
        )).all()
      with benchmark("building cache"):
        cache = defaultdict(lambda: defaultdict(list))
        for rel in relationships:
          if rel.source_type == self.object_class.__name__:
            if rel.destination:
              cache[rel.source_id][rel.destination_type].append(
                  identifier(rel.destination))
          elif rel.source:
            cache[rel.destination_id][rel.source_type].append(
                identifier(rel.source))
      return cache

  def get_mapping_cache(self):
    if self._mapping_cache is None:
      self._mapping_cache = self._create_mapping_cache()
    return self._mapping_cache

  def check_for_duplicate_columns(self, raw_headers):
    counter = Counter(raw_headers)
    duplicates = [header for header, count in counter.items() if count > 1]
    if duplicates:
      self.add_errors(errors.DUPLICATE_COLUMN,
                      line=self.offset + 1,
                      duplicates=", ".join(duplicates))

  def organize_fields(self, fields):
    if fields == "all":
      fields = self.object_headers.keys()
    self.fields = get_column_order(fields)

  def generate_csv_header(self):
    """ Generate 2D array with csv headre description """
    headers = []
    for field in self.fields:
      description = self.object_headers[field]["description"]
      display_name = self.object_headers[field]["display_name"]
      if self.object_headers[field]["mandatory"]:
        display_name += "*"
      headers.append([description, display_name])
    return map(list, zip(*headers))<commit_msg>Patch caching of invalid objects<commit_after># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: <EMAIL>
# Maintained By: <EMAIL>

from collections import defaultdict
from collections import OrderedDict
from collections import Counter

from flask import current_app
from sqlalchemy import or_
from sqlalchemy import and_

from ggrc import db
from ggrc import models
from ggrc.utils import benchmark
from ggrc.converters import errors
from ggrc.converters import get_shared_unique_rules
from ggrc.converters.base_row import RowConverter
from ggrc.converters.import_helper import get_column_order
from ggrc.converters.import_helper import get_object_column_definitions
from ggrc.rbac import permissions
from ggrc.services.common import get_modified_objects
from ggrc.services.common import update_index
from ggrc.services.common import update_memcache_after_commit
from ggrc.services.common import update_memcache_before_commit


CACHE_EXPIRY_IMPORT = 600


class BlockConverter(object):

  """ Main block converter class for dealing with csv files and data

  Attributes:
    attr_index (dict): reverse index for getting attribute name from
      display_name
    block_errors (list of str): list containing fatal import errors
    block_warnings (list of str): list containing blokc level import warnings
    row_errors (list of str): list containing row errors
    row_warnings (list of str): list containing row warnings
    object_ids (list of int): list containing all ids for the converted objects
    rows (list of list of str): 2D array containg csv data
    row_converters (list of RowConverter): list of row convertor objects with
      data from the coresponding row in rows attribute
    object_headers (dict): A dictionary containing object headers
    headers (dict): A dictionary containing csv headers with additional
      information. Keys are object attributes such as "title", "slug"...
      Example:
        {"slug": {
            "display_name": "Code",
            "mandatory": True,
            "default": "value or a callabe function that returns the value",
            "validator": validator_function,
            "handler": HandlerClass,
            "description": "description or function that returns description"
            "valid_values": "list of valid values"

  """

  def get_unique_counts_dict(self, object_class):
    """ get a the varible for storing unique counts

    Make sure to always return the same variable for object with shared tables,
    as defined in sharing rules.
    """
    sharing_rules = get_shared_unique_rules()
    classes = sharing_rules.get(object_class, object_class)
    shared_state = self.converter.shared_state
    if classes not in shared_state:
      shared_state[classes] = defaultdict(lambda: defaultdict(list))
    return shared_state[classes]

  def __init__(self, converter, **options):
    self._mapping_cache = None
    self.converter = converter
    self.offset = options.get("offset", 0)
    self.object_class = options.get("object_class")
    self.rows = options.get("rows", [])
    self.object_ids = options.get("object_ids", [])
    self.block_errors = []
    self.block_warnings = []
    self.row_errors = []
    self.row_warnings = []
    self.row_converters = []
    self.ignore = False
    if not self.object_class:
      class_name = options.get("class_name", "")
      self.add_errors(errors.WRONG_OBJECT_TYPE, line=self.offset + 2,
                      object_name=class_name)
      self.name = ""
      return
    self.object_headers = get_object_column_definitions(self.object_class)
    all_header_names = [unicode(key) for key in self.get_header_names().keys()]
    raw_headers = options.get("raw_headers", all_header_names)
    self.check_for_duplicate_columns(raw_headers)
    self.headers = self.clean_headers(raw_headers)
    self.unique_counts = self.get_unique_counts_dict(self.object_class)
    self.name = self.object_class._inflector.human_singular.title()
    self.organize_fields(options.get("fields", []))
    self.ca_definitions_cache = self._create_ca_definitions_cache()

  def _create_ca_definitions_cache(self):
    cad = models.CustomAttributeDefinition
    definition_type = self.object_class._inflector.table_singular
    defs = cad.eager_query().filter(cad.definition_type == definition_type)
    return {(d.definition_id, d.title): d for d in defs}

  def _create_mapping_cache(self):
    def identifier(obj):
      return getattr(obj, "slug", getattr(obj, "email", None))

    relationship = models.Relationship

    with benchmark("cache for: {}".format(self.object_class.__name__)):
      with benchmark("cache query"):
        relationships = relationship.eager_query().filter(or_(
            and_(
                relationship.source_type == self.object_class.__name__,
                relationship.source_id.in_(self.object_ids),
            ),
            and_(
                relationship.destination_type == self.object_class.__name__,
                relationship.destination_id.in_(self.object_ids),
            )
        )).all()
      with benchmark("building cache"):
        cache = defaultdict(lambda: defaultdict(list))
        for rel in relationships:
          try:
            if rel.source_type == self.object_class.__name__:
              if rel.destination:
                cache[rel.source_id][rel.destination_type].append(
                    identifier(rel.destination))
            elif rel.source:
              cache[rel.destination_id][rel.source_type].append(
                  identifier(rel.source))
          except AttributeError:
            # TODO fix the root cause of the bad relationship object and remove
            # this try except block
            current_app.logger.error("Failed adding object to relationship "
                                     "cache. Rel id: %s", rel.id)
      return cache

  def get_mapping_cache(self):
    if self._mapping_cache is None:
      self._mapping_cache = self._create_mapping_cache()
    return self._mapping_cache

  def check_for_duplicate_columns(self, raw_headers):
    counter = Counter(raw_headers)
    duplicates = [header for header, count in counter.items() if count > 1]
    if duplicates:
      self.add_errors(errors.DUPLICATE_COLUMN,
                      line=self.offset + 1,
                      duplicates=", ".join(duplicates))

  def organize_fields(self, fields):
    if fields == "all":
      fields = self.object_headers.keys()
    self.fields = get_column_order(fields)

  def generate_csv_header(self):
    """ Generate 2D array with csv headre description """
    headers = []
    for field in self.fields:
      description = self.object_headers[field]["description"]
      display_name = self.object_headers[field]["display_name"]
      if self.object_headers[field]["mandatory"]:
        display_name += "*"
      headers.append([description, display_name])
    return map(list, zip(*headers))
{'fraction_non_alphanumeric': 0.08774353009595813, 'fraction_numerical': 0.0021808665309683045, 'mean_word_length': 3.350727387729285, 'pattern_counts': {'":': 18, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2211343', 'n_tokens_mistral': 3912, 'n_tokens_neox': 3794, 'n_words': 1030}