<commit_before># Copyright (C) 2013 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: <EMAIL>
# Maintained By: <EMAIL>

import csv
import chardet
import os
from ggrc.models import Directive, Section
from StringIO import StringIO
from flask import current_app
from ggrc import db
from .common import ImportException
from ggrc.converters.sections import SectionsConverter
from ggrc.converters.controls import ControlsConverter

def handle_csv_import(converter_class, filepath, **options):
  rows = []
  csv_file = filepath
  if isinstance(filepath, basestring):
    csv_file = open(filepath, 'rbU')

  if options.get('directive_id') and not options.get('directive'):
    options['directive'] = Directive.query.filter_by(id=int(options['directive_id'])).first()

  try:
    if isinstance(csv_file, list):
      rows = [row for row in csv_reader(csv_file)]
    else:
      rows = [row for row in csv_reader(csv_file.read().splitlines(True))]
  except UnicodeDecodeError: # Decode error occurs when a special character symbol is inserted in excel.
    raise ImportException("Could not import: invalid character encountered, verify the file is correctly formatted.")
  if not isinstance(csv_file, list):
    csv_file.close()
  converter = converter_class.from_rows(rows, **options)
  # remove 'dry_run' key to allow passing dict w/out keyword arg collision
  # on 'dry_run' parameter:
  is_dry_run = options.get('dry_run', True)
  del options['dry_run']
  return converter.do_import(is_dry_run, **options)

def csv_reader(csv_data, dialect=csv.excel, **kwargs):
  reader = csv.reader(utf_8_encoder(csv_data), dialect=dialect, **kwargs)
  for row in reader:
    yield [unicode(cell, 'utf-8') for cell in row]

def utf_8_encoder(csv_data):
  """This function is a generator that attempts to encode the string as utf-8.
  It is assumed that the data is likely to be encoded in ascii. If encoding
  fails, however, the function will attempt to guess the encoding and convert
  it to utf-8.<commit_msg>Ensure that csv reading error gets ImportException.<commit_after># Copyright (C) 2013 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: <EMAIL>
# Maintained By: <EMAIL>

import csv, _csv
import chardet
import os
from ggrc.models import Directive, Section
from StringIO import StringIO
from flask import current_app
from ggrc import db
from .common import ImportException
from ggrc.converters.sections import SectionsConverter
from ggrc.converters.controls import ControlsConverter

def handle_csv_import(converter_class, filepath, **options):
  rows = []
  csv_file = filepath
  if isinstance(filepath, basestring):
    csv_file = open(filepath, 'rbU')

  if options.get('directive_id') and not options.get('directive'):
    options['directive'] = Directive.query.filter_by(id=int(options['directive_id'])).first()

  try:
    if isinstance(csv_file, list):
      rows = [row for row in csv_reader(csv_file)]
    else:
      rows = [row for row in csv_reader(csv_file.read().splitlines(True))]
  except (UnicodeDecodeError, _csv.Error): # Decode error occurs when a special character symbol is inserted in excel.
    raise ImportException("Could not import: invalid character or spreadsheet encountered; verify the file is correctly formatted.")
  if not isinstance(csv_file, list):
    csv_file.close()
  converter = converter_class.from_rows(rows, **options)
  # remove 'dry_run' key to allow passing dict w/out keyword arg collision
  # on 'dry_run' parameter:
  is_dry_run = options.get('dry_run', True)
  del options['dry_run']
  return converter.do_import(is_dry_run, **options)

def csv_reader(csv_data, dialect=csv.excel, **kwargs):
  reader = csv.reader(utf_8_encoder(csv_data), dialect=dialect, **kwargs)
  for row in reader:
    yield [unicode(cell, 'utf-8') for cell in row]

def utf_8_encoder(csv_data):
  """This function is a generator that attempts to encode the string as utf-8.
  It is assumed that the data is likely to be encoded in ascii. If encoding
  fails, however, the function will attempt to guess the encoding and convert
  it to utf-8.
{'fraction_non_alphanumeric': 0.09631195677707306, 'fraction_numerical': 0.00516795865633075, 'mean_word_length': 4.9221140472878995, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7303529', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1278, 'n_words': 443}